panic: test timed out after 2m0s goroutine 115936 [running]: testing.startAlarm.func1() /Users/andrei/work/src/go/src/testing/testing.go:1145 +0xf9 created by time.goFunc /Users/andrei/work/src/go/src/time/sleep.go:170 +0x44 goroutine 1 [chan receive, 2 minutes]: testing.(*T).Run(0xc4202d8690, 0x5c62843, 0x1c, 0x5d2ad10, 0x408f701) /Users/andrei/work/src/go/src/testing/testing.go:790 +0x2fc testing.runTests.func1(0xc4202d8690) /Users/andrei/work/src/go/src/testing/testing.go:1004 +0x64 testing.tRunner(0xc4202d8690, 0xc420627db8) /Users/andrei/work/src/go/src/testing/testing.go:746 +0xd0 testing.runTests(0xc42048e280, 0x6c9e8c0, 0xd6, 0xd6, 0x0) /Users/andrei/work/src/go/src/testing/testing.go:1002 +0x2d8 testing.(*M).Run(0xc420a2bf18, 0xc420627f30) /Users/andrei/work/src/go/src/testing/testing.go:921 +0x111 github.com/cockroachdb/cockroach/pkg/sql_test.TestMain(0xc420627f18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/main_test.go:36 +0xb8 main.main() github.com/cockroachdb/cockroach/pkg/sql/_test/_testmain.go:478 +0xdb goroutine 8 [chan receive]: github.com/cockroachdb/cockroach/pkg/util/log.flushDaemon() /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1077 +0xf1 created by github.com/cockroachdb/cockroach/pkg/util/log.init.0 /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:582 +0xd9 goroutine 9 [syscall, 2 minutes]: os/signal.signal_recv(0x0) /Users/andrei/work/src/go/src/runtime/sigqueue.go:131 +0xa7 os/signal.loop() /Users/andrei/work/src/go/src/os/signal/signal_unix.go:22 +0x22 created by os/signal.init.0 /Users/andrei/work/src/go/src/os/signal/signal_unix.go:28 +0x41 goroutine 36 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42050a500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc42050a500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 20 [semacquire, 1 minutes]: sync.runtime_Semacquire(0xc421b3e4dc) /Users/andrei/work/src/go/src/runtime/sema.go:56 +0x39 sync.(*WaitGroup).Wait(0xc421b3e4d0) /Users/andrei/work/src/go/src/sync/waitgroup.go:131 +0x72 github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables(0xc4202d8780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:355 +0x376 testing.tRunner(0xc4202d8780, 0x5d2ad10) /Users/andrei/work/src/go/src/testing/testing.go:746 +0xd0 created by testing.(*T).Run /Users/andrei/work/src/go/src/testing/testing.go:789 +0x2de goroutine 21 [select, 2 minutes, locked to thread]: runtime.gopark(0x5d2d048, 0x0, 0x5c3dc68, 0x6, 0x18, 0x1) /Users/andrei/work/src/go/src/runtime/proc.go:277 +0x12c runtime.selectgo(0xc42064ef50, 0xc42007c9c0) /Users/andrei/work/src/go/src/runtime/select.go:395 +0x1138 runtime.ensureSigM.func1() /Users/andrei/work/src/go/src/runtime/signal_unix.go:511 +0x1fe runtime.goexit() /Users/andrei/work/src/go/src/runtime/asm_amd64.s:2337 +0x1 goroutine 22 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc4204ba360, 0xc4200ccba0, 0xc4202dacb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:125 +0x10d created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:123 +0xc9 goroutine 23 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x6a552a0, 0xc4204d8000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:321 +0x51 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043f410, 0xc4204ba360, 0xc420408960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 50 [semacquire, 1 minutes]: sync.runtime_notifyListWait(0xc420066a28, 0xc400000000) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420066a18) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc420066900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:623 +0xa2 created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:610 +0x44d goroutine 51 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420730030) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c250, 0xc4204ba360, 0xc42048c230) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 52 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420730060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c270, 0xc4204ba360, 0xc42048c260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 53 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420730000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c2a0, 0xc4204ba360, 0xc42048c290) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 54 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420732030) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c330, 0xc4204ba360, 0xc42048c2d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 55 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420732000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c360, 0xc4204ba360, 0xc42048c340) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 56 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc4204d8060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c390, 0xc4204ba360, 0xc42048c370) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 57 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc4204d8030) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c3b0, 0xc4204ba360, 0xc42048c3a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 58 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc4204d80c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c3e0, 0xc4204ba360, 0xc42048c3d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 59 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc4204d80f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c430, 0xc4204ba360, 0xc42048c420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 60 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc4207320c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c450, 0xc4204ba360, 0xc42048c440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 61 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc4204d8120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c470, 0xc4204ba360, 0xc42048c460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 62 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc4204d8090) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c490, 0xc4204ba360, 0xc42048c480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 63 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420732090) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c4b0, 0xc4204ba360, 0xc42048c4a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 64 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc4207300c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c4d0, 0xc4204ba360, 0xc42048c4c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 65 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420730090) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c4f0, 0xc4204ba360, 0xc42048c4e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 66 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420732060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c510, 0xc4204ba360, 0xc42048c500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 67 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420721e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1123 +0x68 created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:80 +0x2b8 goroutine 68 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420721f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1123 +0x68 created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:80 +0x2b8 goroutine 69 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x6a552a0, 0xc4207300f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1469 +0x1f3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c530, 0xc4204ba360, 0xc42086e180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 70 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x6a552a0, 0xc4204d8150) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c590, 0xc4204ba360, 0xc420382120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 71 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2(0x6a552a0, 0xc4207320f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:790 +0x53 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c640, 0xc4204ba360, 0xc420382220) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 72 [IO wait, 2 minutes]: internal/poll.runtime_pollWait(0x73cc6c0, 0x72, 0xffffffffffffffff) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712418, 0x72, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712418, 0xffffffffffffff00, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Accept(0xc420712400, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:334 +0x1e2 net.(*netFD).accept(0xc420712400, 0x735b458, 0x0, 0x1) /Users/andrei/work/src/go/src/net/fd_unix.go:238 +0x42 net.(*TCPListener).accept(0xc42000e3c8, 0xc420648ef8, 0x10, 0x4ae8f17) /Users/andrei/work/src/go/src/net/tcpsock_posix.go:136 +0x2e net.(*TCPListener).Accept(0xc42000e3c8, 0x5d2b638, 0xc4203b5e00, 0xc4203a2000, 0xc420648ee0) /Users/andrei/work/src/go/src/net/tcpsock.go:247 +0x49 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc4203b5e00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x95 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func3(0x6a552a0, 0xc4204d8180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:802 +0x2f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c650, 0xc4204ba360, 0xc420382280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 73 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420382240, 0xc4200180f8, 0x59e1ac0, 0x6c7b6c0, 0x5c07060) :1 +0x65 net/http.(*Server).Serve(0xc420882000, 0x6a54460, 0xc420382240, 0x0, 0x0) /Users/andrei/work/src/go/src/net/http/server.go:2695 +0x1b2 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func4(0x6a552a0, 0xc420730120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:814 +0x119 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c670, 0xc4204ba360, 0xc42086e360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 74 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420382260, 0xc420732180, 0xc420729e80, 0x412762a, 0x5b49240) :1 +0x65 crypto/tls.(*listener).Accept(0xc4203822a0, 0xc4200180f8, 0x59e1ac0, 0x6c7b6c0, 0x5c07060) /Users/andrei/work/src/go/src/crypto/tls/tls.go:52 +0x37 net/http.(*Server).Serve(0xc4204f4dd0, 0x6a461e0, 0xc4203822a0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/http/server.go:2695 +0x1b2 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x6a552a0, 0xc420732120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:821 +0x42 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c680, 0xc4204ba360, 0xc4203822c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 75 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x6a552a0, 0xc4204d81b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:825 +0x7c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c690, 0xc4204ba360, 0xc4203b5e40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 76 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4203821c0, 0x5d2c428, 0xc42013e2c0, 0x6a61a80, 0xc4203e8640) :1 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc42013e2c0, 0x6a54460, 0xc4203821c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:504 +0x1ec github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7(0x6a552a0, 0xc4204d81e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:837 +0x43 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c6b0, 0xc4204ba360, 0xc4203822e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 24 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x6a552a0, 0xc4207302a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043e830, 0xc4204ba360, 0xc42037c040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 77 [semacquire]: sync.runtime_notifyListWait(0xc420727f28, 0xc400002885) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420727f18) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc420727e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:623 +0xa2 created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:610 +0x44d goroutine 78 [chan receive, 2 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x6a552a0, 0xc4204d8210) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:371 +0x5a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c910, 0xc4204ba360, 0xc4203824c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 79 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x6a552a0, 0xc4204d8270) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1160 +0x46b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c930, 0xc4204ba360, 0xc42048c920) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 80 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x6a552a0, 0xc4204d82a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1189 +0x269 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c950, 0xc4204ba360, 0xc42048c940) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 496 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420420a00, 0xc4212ac370, 0xc420335310, 0xf, 0xc42007d440, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:581 +0x42c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x6a552a0, 0xc420d86420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:512 +0x74 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ec1e0, 0xc421163e60, 0xc420d863f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 188 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420061840, 0xc420f2f800, 0xc4206cafb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc42038f500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 210 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage/idalloc.(*Allocator).start.func1(0x6a552a0, 0xc42086f2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/idalloc/id_alloc.go:138 +0x6b4 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f2f40, 0xc4204ba360, 0xc4203f2ed0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 189 [IO wait]: internal/poll.runtime_pollWait(0x73cc600, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712298, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712298, 0xc420706800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712280, 0xc420706800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712280, 0xc420706800, 0x400, 0x400, 0x2e, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee020, 0xc420706800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4207321b0, 0x77aa1a0, 0xc4206ee020, 0x5, 0xc4206ee020, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206f8380, 0x5d2d117, 0xc4206f84a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206f8380, 0xc420c9c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420f21920, 0xc42100a1f8, 0x9, 0x9, 0x9, 0x0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420f21920, 0xc42100a1f8, 0x9, 0x9, 0x9, 0xc420757de0, 0x400436e, 0xc420757e8f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420f21920, 0xc42100a1f8, 0x9, 0x9, 0x44f790b, 0xc42250e10c, 0xc4225c6600) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42100a1f8, 0x9, 0x9, 0x6a34360, 0xc420f21920, 0x0, 0x0, 0xc42250e100, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42100a1c0, 0xc42250e100, 0xc42250e100, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4206c0900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 124 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420cbfbf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 629 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005864) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062a20, 0xc421163e60, 0xc421062a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 514 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x6a552a0, 0xc420fb0000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:321 +0x51 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf8560, 0xc421163e60, 0xc420eaaa20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 508 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc4206c0a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1323 +0x14b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 190 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc4208a4e40, 0xc420d87830, 0xc420751fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4206c0900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 465 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc421163e60, 0xc420e1a240, 0xc420b8a770) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:125 +0x10d created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:123 +0xc9 goroutine 126 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420e0c280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420e0c280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 125 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc4208a4d40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 506 [IO wait]: internal/poll.runtime_pollWait(0x73cc300, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420462218, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420462218, 0xc420d3e000, 0x2000, 0x2000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420462200, 0xc420d3e000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420462200, 0xc420d3e000, 0x2000, 0x2000, 0x6a, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0010, 0xc420d3e000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420d68750, 0x77aa1a0, 0xc4205b0010, 0x5, 0xc4205b0010, 0xc420d54a60) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420064700, 0x5d2d117, 0xc420064820, 0x40147f7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420064700, 0xc420c24000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420fd90e0, 0xc420d96ab8, 0x9, 0x9, 0x0, 0x402e408, 0xc420034700) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420fd90e0, 0xc420d96ab8, 0x9, 0x9, 0x9, 0xc424607d10, 0xc420d54e20, 0x454a13e) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420fd90e0, 0xc420d96ab8, 0x9, 0x9, 0x5d2c5d8, 0x200050100, 0xc4000037a7) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420d96ab8, 0x9, 0x9, 0x6a34360, 0xc420fd90e0, 0x0, 0x0, 0x0, 0xc421ac84f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420d96a80, 0xc4239186c0, 0xc4239186c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4206c0a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 515 [semacquire, 1 minutes]: sync.runtime_notifyListWait(0xc420b28d28, 0xc400000000) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420b28d18) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc420b28c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:623 +0xa2 created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:610 +0x44d goroutine 473 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x6a552a0, 0xc420fb02d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062040, 0xc421163e60, 0xc420364040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 191 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc4206c0900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1350 +0x305 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 187 [semacquire]: sync.runtime_notifyListWait(0xc4206b8040, 0xc400000017) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4206b8030) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 io.(*pipe).read(0xc4206b8000, 0xc420534c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:47 +0xc6 io.(*PipeReader).Read(0xc4205b0018, 0xc420534c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:130 +0x4c crypto/tls.(*block).readFromUntil(0xc4204e0960, 0x73ccb68, 0xc420426a50, 0x5, 0xc420426a50, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4202cd500, 0x5d2d117, 0xc4202cd620, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4202cd500, 0xc420ff0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420fd80c0, 0xc420d962d8, 0x9, 0x9, 0xc422166fe0, 0x77ca040, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420fd80c0, 0xc420d962d8, 0x9, 0x9, 0x9, 0xc42074dd60, 0x400436e, 0xc42074de0f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420fd80c0, 0xc420d962d8, 0x9, 0x9, 0x44f790b, 0xc422166fec, 0xc420374050) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420d962d8, 0x9, 0x9, 0x6a34360, 0xc420fd80c0, 0x0, 0x0, 0xc422166fe0, 0xc42074de90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420d962a0, 0xc422166fe0, 0xc422166fe0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4206c0600, 0xc420d86030, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc42013e2c0, 0x6a60d60, 0xc4206c0600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc4203b8000, 0xc42013e2c0, 0x73ccf20, 0xc4206c0600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 181 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc4204e1590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420410400, 0xc4204ba360, 0xc420396520) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 186 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc4206c0600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 182 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc42054e1a0, 0x6a552a0, 0xc4204e1410, 0xc4204ba360, 0x22cd7068, 0xed2292262, 0x0, 0xc420edd500, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:212 +0x226 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc420edd500, 0xc420458901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x87 github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420458900, 0xc422280840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:324 +0x1ba github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x6a552a0, 0xc4204e1410) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420410410, 0xc4204ba360, 0xc420396540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 31 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c51) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0a20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0a20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426e30, 0xc4204ba360, 0xc420426e20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 16 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc4204e14d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204103c0, 0xc4204ba360, 0xc4203964a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 179 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc4204e1530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204103e0, 0xc4204ba360, 0xc4203964e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 178 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc4204e1500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204103d0, 0xc4204ba360, 0xc4203964c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 15 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc4204e14a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204103b0, 0xc4204ba360, 0xc420396480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 180 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc4204e1560) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204103f0, 0xc4204ba360, 0xc420396500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 28 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420163e30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 184 [semacquire]: sync.runtime_notifyListWait(0xc4206b8100, 0xc400000015) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4206b80f0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 io.(*pipe).read(0xc4206b80c0, 0xc420706c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:47 +0xc6 io.(*PipeReader).Read(0xc4205b0028, 0xc420706c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:130 +0x4c crypto/tls.(*block).readFromUntil(0xc42007ef00, 0x73ccb68, 0xc420426a60, 0x5, 0xc420426a60, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206f8000, 0x5d2d117, 0xc4206f8120, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206f8000, 0xc420fe6000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420fd8060, 0xc42040fdf8, 0x9, 0x9, 0xc420886480, 0x4, 0xc420a27d88) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420fd8060, 0xc42040fdf8, 0x9, 0x9, 0x9, 0xc420a27e18, 0x3, 0xc420a27e8f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420fd8060, 0xc42040fdf8, 0x9, 0x9, 0x44f790b, 0xc422166fcc, 0xc420d810d8) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42040fdf8, 0x9, 0x9, 0x6a34360, 0xc420fd8060, 0x0, 0x0, 0xc420f2f808, 0xc420a27ed0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42040fdc0, 0xc422166fc0, 0xc422166fc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc42038f500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 14 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc4204e1470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204103a0, 0xc4204ba360, 0xc420396460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 13 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc4204e1440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420410390, 0xc4204ba360, 0xc420396440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 26 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func12(0x6a552a0, 0xc420159110) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1073 +0x7c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426a70, 0xc4204ba360, 0xc420163da0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 29 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func15(0x6a552a0, 0xc4204e0990) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1103 +0x53 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426b50, 0xc4204ba360, 0xc4203a4420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 491 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4203821a0, 0x5d2b520, 0xc4204ba360, 0x6a552a0, 0xc420cbeb10) :1 +0x65 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc42000e3a0, 0x6a552a0, 0xc420cbeb10, 0xc4204ba360, 0x6a54460, 0xc4203821a0, 0xc420cbeb40, 0xc420cbeb10, 0xc420a786d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:119 +0x50 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17(0x6a552a0, 0xc420cbeb10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1277 +0x1f5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043fd30, 0xc4204ba360, 0xc420f0e4c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 25 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73cc780, 0x72, 0xffffffffffffffff) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712398, 0x72, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712398, 0xffffffffffffff00, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Accept(0xc420712380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:334 +0x1e2 net.(*netFD).accept(0xc420712380, 0xc423f81330, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:238 +0x42 net.(*TCPListener).accept(0xc42000e3b0, 0x4ae900a, 0x405c590, 0xc420748e80) /Users/andrei/work/src/go/src/net/tcpsock_posix.go:136 +0x2e net.(*TCPListener).Accept(0xc42000e3b0, 0x5d2b640, 0xc4203b5dc0, 0x6a61c00, 0xc423f81330) /Users/andrei/work/src/go/src/net/tcpsock.go:247 +0x49 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc4203b5dc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x95 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func11.1() /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1032 +0x2f sync.(*Once).Do(0xc4203ccf60, 0xc420892758) /Users/andrei/work/src/go/src/sync/once.go:44 +0xbe github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func11(0x6a552a0, 0xc4201590e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1031 +0x5b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204269e0, 0xc4204ba360, 0xc4203a4220) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 30 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x6a552a0, 0xc4204e09c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x49 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426e10, 0xc4204ba360, 0xc4203a4f40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 582 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420359080, 0x5d2b520, 0xc421163e60, 0x6a552a0, 0xc420e49ef0) :1 +0x65 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc4206ee360, 0x6a552a0, 0xc420e49ef0, 0xc421163e60, 0x6a54460, 0xc420359080, 0xc420e49f20, 0xc420e49ef0, 0xc420bcced0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:119 +0x50 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17(0x6a552a0, 0xc420e49ef0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1277 +0x1f5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378400, 0xc421163e60, 0xc420f0e480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 40 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc42086f7a0, 0xc42086f620, 0xc420755fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc420f66a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 35 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420061580) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 32 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c47) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0a50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0a50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426e50, 0xc4204ba360, 0xc420426e40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 41 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc420f66a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 33 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c4a) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426e70, 0xc4204ba360, 0xc420426e60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 130 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c54) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0ab0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0ab0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426e90, 0xc4204ba360, 0xc420426e80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 131 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c48) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e09f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e09f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426eb0, 0xc4204ba360, 0xc420426ea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 132 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c50) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426ed0, 0xc4204ba360, 0xc420426ec0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 133 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c4c) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0b10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0b10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426ef0, 0xc4204ba360, 0xc420426ee0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 134 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c55) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0b70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0b70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426f10, 0xc4204ba360, 0xc420426f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 135 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c59) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0ba0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0ba0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426f30, 0xc4204ba360, 0xc420426f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 136 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c42) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0bd0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0bd0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426f50, 0xc4204ba360, 0xc420426f40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 137 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c46) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d98f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d98f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426f70, 0xc4204ba360, 0xc420426f60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 138 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c57) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d98c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d98c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426f90, 0xc4204ba360, 0xc420426f80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 139 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c44) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427640, 0xc4204ba360, 0xc420426fb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 140 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c40) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427680, 0xc4204ba360, 0xc420427670) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 141 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c41) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9920) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9920) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204276b0, 0xc4204ba360, 0xc420427690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 142 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c3d) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d99e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d99e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427740, 0xc4204ba360, 0xc420427730) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 143 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c58) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d99b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d99b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427760, 0xc4204ba360, 0xc420427750) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 144 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c43) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427780, 0xc4204ba360, 0xc420427770) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 145 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c45) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204277a0, 0xc4204ba360, 0xc420427790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 146 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c53) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9aa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9aa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204277c0, 0xc4204ba360, 0xc4204277b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 147 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c3b) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9ad0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9ad0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204277e0, 0xc4204ba360, 0xc4204277d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 148 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c3f) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9a70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9a70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427810, 0xc4204ba360, 0xc4204277f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 149 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c3c) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9b30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9b30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427830, 0xc4204ba360, 0xc420427820) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 150 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c4b) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9b00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9b00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427860, 0xc4204ba360, 0xc420427850) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 151 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c4f) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427890, 0xc4204ba360, 0xc420427880) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 152 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c49) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204d9b90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204d9b90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204278c0, 0xc4204ba360, 0xc4204278b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 153 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c3e) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204278f0, 0xc4204ba360, 0xc4204278e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 154 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c52) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0c30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0c30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427920, 0xc4204ba360, 0xc420427900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 155 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c4e) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0c90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0c90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427950, 0xc4204ba360, 0xc420427930) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 156 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c56) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427980, 0xc4204ba360, 0xc420427960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 157 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c4d) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0cf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0cf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427a10, 0xc4204ba360, 0xc420427990) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 158 [semacquire]: sync.runtime_notifyListWait(0xc4207479d0, 0xc400006c3a) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4207479c0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc42088a120, 0x6a552a0, 0xc4204e0cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4204e0cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427a30, 0xc4204ba360, 0xc420427a20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 159 [semacquire, 2 minutes]: sync.runtime_Semacquire(0xc42088a1a4) /Users/andrei/work/src/go/src/runtime/sema.go:56 +0x39 sync.(*WaitGroup).Wait(0xc42088a198) /Users/andrei/work/src/go/src/sync/waitgroup.go:131 +0x72 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc42088a120, 0x6a552a0, 0xc4204e0d20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x31 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x6a552a0, 0xc4204e0d20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3875 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427a50, 0xc4204ba360, 0xc420427a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 160 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop(0xc420626400, 0x6a552a0, 0xc4204e0d50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3891 +0x205 github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.raftTickLoop)-fm(0x6a552a0, 0xc4204e0d50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3877 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427a70, 0xc4204ba360, 0xc420427a60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 161 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).coalescedHeartbeatsLoop(0xc420626400, 0x6a552a0, 0xc4204e1320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3932 +0x16d github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.coalescedHeartbeatsLoop)-fm(0x6a552a0, 0xc4204e1320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3878 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427aa0, 0xc4204ba360, 0xc420427a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 162 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x6a552a0, 0xc4204e1350) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1324 +0x122 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427ae0, 0xc4204ba360, 0xc4203a4f60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 163 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc4204e0d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427af0, 0xc4204ba360, 0xc4207346e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 164 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc4204e13b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427b10, 0xc4204ba360, 0xc420734730) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 165 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc4204e1380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427b20, 0xc4204ba360, 0xc420734780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 185 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc420d86cf0, 0xc420d86c90, 0xc4206c8fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc4206c0600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 167 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/storage/compactor.(*Compactor).Start.func1(0x6a552a0, 0xc4204e16b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/compactor/compactor.go:140 +0x1ce github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427bd0, 0xc4204ba360, 0xc420730780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 562 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420e0c780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420e0c780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 169 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x6a552a0, 0xc4204d9bf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:736 +0x14f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427fe0, 0xc4204ba360, 0xc4203a52a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 170 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x6a552a0, 0xc4204d9ce0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:698 +0x2dd github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420427ff0, 0xc4204ba360, 0xc4203a52c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 171 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x6a552a0, 0xc4204d9d40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x2db github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f2060, 0xc4204ba360, 0xc4203a5320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 172 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1(0x6a552a0, 0xc4204d9d70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1465 +0x16c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f2100, 0xc4204ba360, 0xc4203a5340) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 173 [select]: github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x6a552a0, 0xc4204d9da0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:126 +0x156 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f2130, 0xc4204ba360, 0xc4203f2120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 174 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1(0x6a552a0, 0xc4204e19b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:773 +0x1f6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f2170, 0xc4204ba360, 0xc4203a5360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 175 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x6a552a0, 0xc4204d9cb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:1031 +0xa6b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f21a0, 0xc4204ba360, 0xc4203a5380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 176 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*Executor).Start.func1(0x6a552a0, 0xc420731740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:397 +0x129 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f21b0, 0xc4204ba360, 0xc4203a53c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 700 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fa40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203dbfa0, 0xc420f747e0, 0xc4203dbf70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 507 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420746700, 0xc420d69530, 0xc420d53fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4206c0a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 195 [select, 2 minutes]: github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x6a552a0, 0xc4207317a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x17b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f2200, 0xc4204ba360, 0xc4203f21f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 196 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*Server).Start.func1(0x6a552a0, 0xc4207317d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:286 +0x133 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f2210, 0xc4204ba360, 0xc4203a53e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 197 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x6a552a0, 0xc420731c20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:446 +0x2a9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f22a0, 0xc4204ba360, 0xc420734b90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 198 [select]: github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func1(0x6a552a0, 0xc42086f080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:205 +0x165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f22d0, 0xc4204ba360, 0xc420730ed0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 199 [select]: github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x6a552a0, 0xc42086f0b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:216 +0x13b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f22e0, 0xc4204ba360, 0xc420730f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 631 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005862) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e493e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e493e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062ba0, 0xc421163e60, 0xc421062af0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 630 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005860) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e493b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e493b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062ab0, 0xc421163e60, 0xc421062a30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 588 [chan receive, 1 minutes]: database/sql.(*DB).connectionOpener(0xc420f44aa0) /Users/andrei/work/src/go/src/database/sql/sql.go:871 +0x53 created by database/sql.Open /Users/andrei/work/src/go/src/database/sql/sql.go:609 +0x1ee goroutine 701 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7faa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203dbff0, 0xc420f747e0, 0xc4203dbfc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 42 [IO wait]: internal/poll.runtime_pollWait(0x73cc540, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420f42018, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420f42018, 0xc420535800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420f42000, 0xc420535800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420f42000, 0xc420535800, 0x400, 0x400, 0x301, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee018, 0xc420535800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420f44000, 0xc420535800, 0x400, 0x400, 0x5b473a0, 0xc420f2e500, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420f2e510, 0x73ccbf8, 0xc420f44000, 0x5, 0xc420f44000, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420f84000, 0x5d2d117, 0xc420f84120, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420f84000, 0xc420cb4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420c76a20, 0xc420d96578, 0x9, 0x9, 0x9, 0x0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420c76a20, 0xc420d96578, 0x9, 0x9, 0x9, 0xc4206ccd60, 0x400436e, 0xc4206cce0f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420c76a20, 0xc420d96578, 0x9, 0x9, 0x44f790b, 0xc42250e12c, 0xc420cfc000) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420d96578, 0x9, 0x9, 0x6a34360, 0xc420c76a20, 0x0, 0x0, 0xc42250e120, 0xc4206cce90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420d96540, 0xc42250e120, 0xc42250e120, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc420f66a80, 0xc4207328a0, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc42013e2c0, 0x6a60d60, 0xc420f66a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc4203c2c00, 0xc42013e2c0, 0x73ccf20, 0xc420f66a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 494 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc42104e340, 0x6a552a0, 0xc420d862d0, 0xc420b28a80, 0x6a61240, 0xc420bf9f30, 0xc421163e60, 0xc42035e170, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:323 +0x350 github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x6a552a0, 0xc420d862d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:130 +0x3cd github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ec070, 0xc421163e60, 0xc4207462c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 477 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc420d693b0, 0xc420d69350, 0xc420e37fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc4206c0780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 478 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc4206c0780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 698 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7f9e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203dbf30, 0xc420f747e0, 0xc4203dbf20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1251 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420739e40, 0xc4219b8bd0, 0xc4213c8fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4219bc480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 697 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7f9b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203dbf10, 0xc420f747e0, 0xc4203dbf00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 497 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc4207465c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 480 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc4202d8a50, 0x6a612a0, 0xc4210622c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:185 +0x621 github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x5be8da0, 0xc4202d8a50, 0x6a5eea0, 0xc420f44780, 0x4014f58, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:208 +0xb2 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5be8da0, 0xc4202d8a50, 0x6a5eea0, 0xc420f44780, 0xc4203648c0, 0x5d27e48, 0x3f, 0xc420d94300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x103 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc42013e2c0, 0x6a60d60, 0xc4206c0780, 0xc421068b40, 0xc420163440, 0x6c7fb20, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1060 +0x3d6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42013e2c0, 0x6a60d60, 0xc4206c0780, 0xc421068b40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1147 +0x14c1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203747b0, 0xc42013e2c0, 0x6a60d60, 0xc4206c0780, 0xc421068b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 501 [chan receive, 1 minutes]: database/sql.(*DB).connectionOpener(0xc420e09b80) /Users/andrei/work/src/go/src/database/sql/sql.go:871 +0x53 created by database/sql.Open /Users/andrei/work/src/go/src/database/sql/sql.go:609 +0x1ee goroutine 495 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420d863c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 479 [IO wait]: internal/poll.runtime_pollWait(0x73cc240, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712918, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712918, 0xc422ed4000, 0x8000, 0x8000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712900, 0xc422ed4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712900, 0xc422ed4000, 0x8000, 0x8000, 0x7f01, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee3a0, 0xc422ed4000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421051220, 0xc422ed4000, 0x8000, 0x8000, 0x5b473a0, 0xc420d68d00, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420d68de0, 0x73ccbf8, 0xc421051220, 0x5, 0xc421051220, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4202cd180, 0x5d2d117, 0xc4202cd2a0, 0x4007d18) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4202cd180, 0xc420f86000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420fd8fc0, 0xc420d968f8, 0x9, 0x9, 0xc42391da40, 0x1000f0120c2f500, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420fd8fc0, 0xc420d968f8, 0x9, 0x9, 0x9, 0x45472e6, 0xc420db8270, 0xc4229be8a0) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420fd8fc0, 0xc420d968f8, 0x9, 0x9, 0xc4229be8a0, 0x5a, 0x5a) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420d968f8, 0x9, 0x9, 0x6a34360, 0xc420fd8fc0, 0x0, 0x100000000, 0x2b0000005a, 0x5a00000001) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420d968c0, 0xc420d69260, 0xc420d69260, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4206c0780, 0xc420fb13b0, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc42013e2c0, 0x6a60d60, 0xc4206c0780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc420364780, 0xc42013e2c0, 0x73ccf20, 0xc4206c0780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 699 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fa10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203dbf60, 0xc420f747e0, 0xc4203dbf40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 628 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005867) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49350) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49350) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062a00, 0xc421163e60, 0xc4210629f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 626 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x6a552a0, 0xc420e492f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x49 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062930, 0xc421163e60, 0xc4203c33e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 516 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d68030) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9940, 0xc421163e60, 0xc420bf9930) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 517 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420fb0090) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9960, 0xc421163e60, 0xc420bf9950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 518 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d68000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9980, 0xc421163e60, 0xc420bf9970) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 519 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420fb0060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf99a0, 0xc421163e60, 0xc420bf9990) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 520 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d86000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf99c0, 0xc421163e60, 0xc420bf99b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 521 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420fb0030) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf99e0, 0xc421163e60, 0xc420bf99d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 522 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d860f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9a00, 0xc421163e60, 0xc420bf99f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 523 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d86120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9a20, 0xc421163e60, 0xc420bf9a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 524 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d860c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9a40, 0xc421163e60, 0xc420bf9a30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 525 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d68090) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9a60, 0xc421163e60, 0xc420bf9a50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 526 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d680c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9a80, 0xc421163e60, 0xc420bf9a70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 527 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d68060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9aa0, 0xc421163e60, 0xc420bf9a90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 528 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420fb00f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9ac0, 0xc421163e60, 0xc420bf9ab0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 529 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420fb0120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9ae0, 0xc421163e60, 0xc420bf9ad0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 530 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420fb00c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9b00, 0xc421163e60, 0xc420bf9af0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 531 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420d680f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9b20, 0xc421163e60, 0xc420bf9b10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 532 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420f2ca00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1123 +0x68 created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:80 +0x2b8 goroutine 533 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc420f2cb00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1123 +0x68 created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:80 +0x2b8 goroutine 534 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x6a552a0, 0xc420d86150) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1469 +0x1f3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9b30, 0xc421163e60, 0xc420b26cf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 535 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x6a552a0, 0xc420d861b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9b90, 0xc421163e60, 0xc420358fe0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 536 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2(0x6a552a0, 0xc420fb0300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:790 +0x53 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9c10, 0xc421163e60, 0xc420359120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 537 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73cc3c0, 0x72, 0xffffffffffffffff) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712798, 0x72, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712798, 0xffffffffffffff00, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Accept(0xc420712780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:334 +0x1e2 net.(*netFD).accept(0xc420712780, 0x735ad90, 0x0, 0xc4203cb701) /Users/andrei/work/src/go/src/net/fd_unix.go:238 +0x42 net.(*TCPListener).accept(0xc4206ee388, 0xc420649ef8, 0x10, 0x4ae8f17) /Users/andrei/work/src/go/src/net/tcpsock_posix.go:136 +0x2e net.(*TCPListener).Accept(0xc4206ee388, 0x5d2b638, 0xc4211bb180, 0xc420374010, 0x0) /Users/andrei/work/src/go/src/net/tcpsock.go:247 +0x49 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc4211bb180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x95 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func3(0x6a552a0, 0xc420fb0330) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:802 +0x2f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9c20, 0xc421163e60, 0xc420359180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 538 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420359140, 0xc4200180f8, 0x59e1ac0, 0x6c7b6c0, 0x5c07060) :1 +0x65 net/http.(*Server).Serve(0xc42088e000, 0x6a54460, 0xc420359140, 0x0, 0x0) /Users/andrei/work/src/go/src/net/http/server.go:2695 +0x1b2 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func4(0x6a552a0, 0xc420fb0150) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:814 +0x119 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9c30, 0xc421163e60, 0xc420b26f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 539 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc420359160, 0xc420fb03c0, 0xc42064ce80, 0x412762a, 0x5b49240) :1 +0x65 crypto/tls.(*listener).Accept(0xc4203591a0, 0xc4200180f8, 0x59e1ac0, 0x6c7b6c0, 0x5c07060) /Users/andrei/work/src/go/src/crypto/tls/tls.go:52 +0x37 net/http.(*Server).Serve(0xc420ced520, 0x6a461e0, 0xc4203591a0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/http/server.go:2695 +0x1b2 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x6a552a0, 0xc420fb0360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:821 +0x42 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9c40, 0xc421163e60, 0xc4203591c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 540 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x6a552a0, 0xc420fb03f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:825 +0x7c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9c50, 0xc421163e60, 0xc4211bb1c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 541 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4203590c0, 0x5d2c428, 0xc4205e8420, 0x6a61a80, 0xc422ca8be0) :1 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc4205e8420, 0x6a54460, 0xc4203590c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:504 +0x1ec github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7(0x6a552a0, 0xc420fb0420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:837 +0x43 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9c60, 0xc421163e60, 0xc4203591e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 542 [semacquire]: sync.runtime_notifyListWait(0xc420be0728, 0xc40000216b) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420be0718) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc420be0600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:623 +0xa2 created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:610 +0x44d goroutine 543 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x6a552a0, 0xc420fb0450) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:371 +0x5a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9e50, 0xc421163e60, 0xc420359520) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 544 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x6a552a0, 0xc420d86210) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1160 +0x46b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9e70, 0xc421163e60, 0xc420bf9e60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 545 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x6a552a0, 0xc420fb04b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1189 +0x269 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9e90, 0xc421163e60, 0xc420bf9e80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 546 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73cc480, 0x72, 0xffffffffffffffff) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712718, 0x72, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712718, 0xffffffffffffff00, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Accept(0xc420712700, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:334 +0x1e2 net.(*netFD).accept(0xc420712700, 0xc4206ef2c0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:238 +0x42 net.(*TCPListener).accept(0xc4206ee370, 0x4ae900a, 0x405c590, 0xc420d51e80) /Users/andrei/work/src/go/src/net/tcpsock_posix.go:136 +0x2e net.(*TCPListener).Accept(0xc4206ee370, 0x5d2b640, 0xc4211bb140, 0x6a61c00, 0xc4206ef2c0) /Users/andrei/work/src/go/src/net/tcpsock.go:247 +0x49 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc4211bb140, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x95 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10.1() /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1011 +0x2f sync.(*Once).Do(0xc420019a70, 0xc420a7c758) /Users/andrei/work/src/go/src/sync/once.go:44 +0xbe github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10(0x6a552a0, 0xc420b271a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1010 +0x5b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9ec0, 0xc421163e60, 0xc420359560) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 549 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func4(0x6a61b40, 0xc4206c0a80, 0xc420e0c500, 0xc420c79560, 0xc420b00dc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:284 +0x1cd created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:283 +0xaaa goroutine 550 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc421093590, 0xc4203597b0, 0x5, 0x5, 0xc4225b3112, 0xc420f4eaa8, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc421093590, 0xc4203597b0, 0x5, 0x5, 0x3, 0xc420026000, 0xc420026070) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420b27bf0, 0xc4203597b0, 0x5, 0x5, 0x407883e, 0xc420b00eb8, 0xc423a465c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc420b27bf0, 0xc4203597b0, 0x5, 0x5, 0x5, 0xc4206c0a80, 0xc420b00dc0, 0x5) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc420b27bf0, 0xc4203597b0, 0x5, 0x5, 0x6c8ee00, 0xc4225b3110, 0xc420f4ecd0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420b00dc0, 0xc4203597b0, 0x5, 0x5, 0x32, 0x600, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203597a0, 0x7fffffff, 0xc420f4ece0, 0xc423a465c0, 0xc420f4ed50, 0x40147f7, 0xc4238c0420, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203597a0, 0x6a54aa0, 0x6cf0d10, 0xc420b00dc0, 0x0, 0x0, 0x5b26a80, 0xc4237283c0, 0x7fffffff, 0xc4238c0420, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420c79560, 0x5b26a80, 0xc4237283c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:442 +0x10c github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc420bf9f30, 0x6a552a0, 0xc420b27ce0, 0xc420b28a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:191 +0x62 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x6a61240, 0xc420bf9f30, 0xc42104e340, 0x6a552a0, 0xc420b27ce0, 0xc420b28a80, 0xc420a75f70, 0x402dd49) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:311 +0x35 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x6a552a0, 0xc420b27ce0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:319 +0xc0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9fa0, 0xc421163e60, 0xc4211bb3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 551 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc42102bbd0, 0xc4203648b0, 0x5, 0x5, 0x404ab68, 0xc4262e97d0, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42102bbd0, 0xc4203648b0, 0x5, 0x5, 0x3, 0xc420029900, 0xc420029970) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420fb15c0, 0xc4203648b0, 0x5, 0x5, 0x407883e, 0xc421068c38, 0xc4226c5200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc420fb15c0, 0xc4203648b0, 0x5, 0x5, 0x5, 0xc4206c0780, 0xc421068b40, 0xc400000005) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc420fb15c0, 0xc4203648b0, 0x5, 0x5, 0xc4262e98e8, 0x4012f13, 0x9) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc421068b40, 0xc4203648b0, 0x5, 0x5, 0x6a43360, 0xc420029988, 0xc420029900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203648a0, 0x7fffffff, 0x4008553, 0xc42126e118, 0xc4262e9a78, 0x40147f7, 0xc4242a9980, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203648a0, 0x6a54aa0, 0x6cf0d10, 0xc421068b40, 0x0, 0x0, 0x5b42a20, 0xc420b963c0, 0x7fffffff, 0xc4242a9980, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420f44780, 0x5b42a20, 0xc420b963c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:698 +0x12e github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc4210622c0, 0xc4262e9bb0, 0x41a6460, 0xc4202d8a98) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:227 +0x62 github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc4202d8a98, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:148 +0x2f github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc4202d8a50, 0x6a552a0, 0xc420d69c50, 0xc4204f80e8, 0xc4211bb440, 0xc4262e9f58, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:326 +0x474 github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1(0x6a552a0, 0xc420d69c50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:148 +0x96 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9fd0, 0xc4204ba360, 0xc4211bb480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 620 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x6a552a0, 0xc421012180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420410120, 0xc420f747e0, 0xc4203b8060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 613 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc42086ed80, 0xc42086ec90, 0xc4217affb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc4210f2180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 627 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005871) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210629e0, 0xc421163e60, 0xc4210629d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 573 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc420019a74, 0xc420fef400) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc420019a70) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee sync.(*Once).Do(0xc420019a70, 0xc420a83758) /Users/andrei/work/src/go/src/sync/once.go:40 +0x44 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func11(0x6a552a0, 0xc420d87080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1031 +0x5b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ec430, 0xc421163e60, 0xc42037c740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 574 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func12(0x6a552a0, 0xc420d870b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1073 +0x7c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ec490, 0xc421163e60, 0xc420d86e40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 579 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420746cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 576 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420d86ed0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 577 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func15(0x6a552a0, 0xc420d87050) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1103 +0x53 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ec560, 0xc421163e60, 0xc42037c960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 616 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420746e40, 0xc42086e900, 0xc4210a9fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4210f2000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 580 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420e0cc80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420e0cc80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 612 [semacquire]: sync.runtime_notifyListWait(0xc4206b84c0, 0xc400000016) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4206b84b0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 io.(*pipe).read(0xc4206b8480, 0xc4204b1000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:47 +0xc6 io.(*PipeReader).Read(0xc42000e268, 0xc4204b1000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:130 +0x4c crypto/tls.(*block).readFromUntil(0xc420d87320, 0x73ccb68, 0xc4210ec480, 0x5, 0xc4210ec480, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420edc700, 0x5d2d117, 0xc420edc820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420edc700, 0xc421250000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420b3c3c0, 0xc420d96c78, 0x9, 0x9, 0xc420b47380, 0x4, 0xc4217add88) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420b3c3c0, 0xc420d96c78, 0x9, 0x9, 0x9, 0xc4217ade18, 0x3, 0xc4217ade8f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420b3c3c0, 0xc420d96c78, 0x9, 0x9, 0x44f790b, 0xc42310b2ec, 0xc420b3c478) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420d96c78, 0x9, 0x9, 0x6a34360, 0xc420b3c3c0, 0x0, 0x0, 0xc42086e908, 0xc4217aded0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420d96c40, 0xc42310b2e0, 0xc42310b2e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4210f2000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 597 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x6a552a0, 0xc420cbf020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:736 +0x14f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210627d0, 0xc421163e60, 0xc420365c20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 598 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x6a552a0, 0xc420cbf050) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:698 +0x2dd github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210627e0, 0xc421163e60, 0xc420365c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 599 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x6a552a0, 0xc420cbf080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x2db github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062810, 0xc421163e60, 0xc420365c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 600 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1(0x6a552a0, 0xc420cbf0e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1465 +0x16c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062880, 0xc421163e60, 0xc420365c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 601 [select]: github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x6a552a0, 0xc420cbf110) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:126 +0x156 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210628a0, 0xc421163e60, 0xc421062890) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 602 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1(0x6a552a0, 0xc420d86240) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:773 +0x1f6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210628b0, 0xc421163e60, 0xc420365cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 603 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x6a552a0, 0xc420d86660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:1031 +0xa6b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210628c0, 0xc421163e60, 0xc420365ce0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 604 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*Executor).Start.func1(0x6a552a0, 0xc420e487e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:397 +0x129 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210628e0, 0xc421163e60, 0xc420365d00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 696 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7f980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203dbef0, 0xc420f747e0, 0xc4203dbee0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 694 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7f920) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203dbeb0, 0xc420f747e0, 0xc4203dbea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 607 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x6a552a0, 0xc420cbecf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x17b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062960, 0xc421163e60, 0xc421062950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 608 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*Server).Start.func1(0x6a552a0, 0xc420e48810) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:286 +0x133 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210629b0, 0xc421163e60, 0xc420365d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 609 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x6a552a0, 0xc420cbe000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:446 +0x2a9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062a40, 0xc421163e60, 0xc420ba68c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 610 [select]: github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func1(0x6a552a0, 0xc420e48840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:205 +0x165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062a50, 0xc421163e60, 0xc420cbe1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 611 [select]: github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x6a552a0, 0xc420e488a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:216 +0x13b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062a60, 0xc421163e60, 0xc420cbe240) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 632 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005869) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49410) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49410) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062c40, 0xc421163e60, 0xc421062bd0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 633 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005865) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062c70, 0xc421163e60, 0xc421062c50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 634 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000585c) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062cb0, 0xc421163e60, 0xc421062c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 635 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005872) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e494a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e494a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062cd0, 0xc421163e60, 0xc421062cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 636 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000586f) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e494d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e494d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062cf0, 0xc421163e60, 0xc421062ce0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 637 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000585f) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062d10, 0xc421163e60, 0xc421062d00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 638 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005868) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062d30, 0xc421163e60, 0xc421062d20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 639 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005874) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49560) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49560) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062d50, 0xc421163e60, 0xc421062d40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 640 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005866) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062d70, 0xc421163e60, 0xc421062d60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 641 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000586c) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e495c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e495c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062d90, 0xc421163e60, 0xc421062d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 642 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000586b) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e495f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e495f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062db0, 0xc421163e60, 0xc421062da0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 643 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005870) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062dd0, 0xc421163e60, 0xc421062dc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 644 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000585a) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49650) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49650) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062df0, 0xc421163e60, 0xc421062de0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 645 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005859) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062e20, 0xc421163e60, 0xc421062e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 646 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005858) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e496e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e496e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062e40, 0xc421163e60, 0xc421062e30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 647 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000586d) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49710) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49710) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062e60, 0xc421163e60, 0xc421062e50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 648 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005875) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062e90, 0xc421163e60, 0xc421062e80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 649 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000585d) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49770) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49770) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062eb0, 0xc421163e60, 0xc421062ea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 650 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005861) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e497a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e497a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062ed0, 0xc421163e60, 0xc421062ec0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 651 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000585b) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e497d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e497d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062ef0, 0xc421163e60, 0xc421062ee0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 652 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005873) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062f10, 0xc421163e60, 0xc421062f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 653 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005863) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062f30, 0xc421163e60, 0xc421062f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 654 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000586e) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49890) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49890) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062f50, 0xc421163e60, 0xc421062f40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 655 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000586a) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e498c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e498c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062f70, 0xc421163e60, 0xc421062f60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 656 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc40000585e) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e498f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e498f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062f90, 0xc421163e60, 0xc421062f80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 657 [semacquire]: sync.runtime_notifyListWait(0xc4211bb750, 0xc400005876) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211bb740) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49920) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49920) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062fb0, 0xc421163e60, 0xc421062fa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 658 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421180900, 0xc4206a4230, 0x7, 0xc4212baa00, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421180900, 0x7, 0xc4212baa00, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421180900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421180900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421180900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421180900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423226c40, 0x6a552a0, 0xc4211808d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc423fbe6f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc423fbe6f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc423fbe6f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc423fbe6f0, 0xc423f73188, 0xc422a08300, 0x0, 0xc421228b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc423fbe6f0, 0xc422a08300, 0x1, 0xc421228b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc423fbe6f0, 0xc4247d9b00, 0x30, 0x30, 0x15179d5722b53018, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc423fbe6f0, 0xc4247d9b00, 0x30, 0x30, 0x15179d5722b53018, 0x0, 0x2, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc423fbe6f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).runAsyncTask(0xc421013680, 0x6a552a0, 0xc423fbe6f0, 0xc4224e6700, 0x1, 0xc421180840, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:302 +0x1e4 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync(0xc421013680, 0x6a552a0, 0xc423fbe6f0, 0xc4224e6700, 0xc421180780, 0x1, 0x1, 0xc4224e6701, 0x0, 0x15179d5722576460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:318 +0x1af github.com/cockroachdb/cockroach/pkg/storage.(*Replica).loadSystemConfig(0xc4224e6700, 0x6a552a0, 0xc423fbe6f0, 0x8, 0x1, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:5875 +0x411 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).MaybeGossipSystemConfig(0xc4224e6700, 0x6a552a0, 0xc423fbe6f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:5744 +0xe3 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleLocalEvalResult(0xc4224e6700, 0x6a552a0, 0xc423fbe6f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x100, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:752 +0x4c2 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleEvalResultRaftMuLocked(0xc4224e6700, 0x6a552a0, 0xc423fbe6f0, 0xc422d37980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica_proposal.go:794 +0x1ab github.com/cockroachdb/cockroach/pkg/storage.(*Replica).processRaftCommand(0xc4224e6700, 0x6a552a0, 0xc423fbe6f0, 0xc424e377e8, 0x8, 0x7, 0x33df, 0x200000002, 0x3, 0x2, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:4791 +0x910 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).handleRaftReadyRaftMuLocked(0xc4224e6700, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3640 +0x1337 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue.func1(0x6a552a0, 0xc421f531a0, 0xc4224e6700, 0x6a552a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3774 +0x109 github.com/cockroachdb/cockroach/pkg/storage.(*Store).withReplicaForRequest(0xc4206d1c00, 0x6a552a0, 0xc421f531a0, 0xc4212456c0, 0xc423f75ec8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3115 +0x135 github.com/cockroachdb/cockroach/pkg/storage.(*Store).processRequestQueue(0xc4206d1c00, 0x6a552a0, 0xc420e49950, 0x7) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3762 +0x24f github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc4211a7170, 0x6a552a0, 0xc420e49950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:226 +0x21b github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e49950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062fd0, 0xc421163e60, 0xc421062fc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 659 [semacquire, 1 minutes]: sync.runtime_Semacquire(0xc4211a71f4) /Users/andrei/work/src/go/src/runtime/sema.go:56 +0x39 sync.(*WaitGroup).Wait(0xc4211a71e8) /Users/andrei/work/src/go/src/sync/waitgroup.go:131 +0x72 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc4211a7170, 0x6a552a0, 0xc420e49980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x31 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x6a552a0, 0xc420e49980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3875 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421062ff0, 0xc421163e60, 0xc421062fe0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 660 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop(0xc4206d1c00, 0x6a552a0, 0xc420e499b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3891 +0x205 github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.raftTickLoop)-fm(0x6a552a0, 0xc420e499b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3877 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063010, 0xc421163e60, 0xc421063000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 661 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).coalescedHeartbeatsLoop(0xc4206d1c00, 0x6a552a0, 0xc420e499e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3932 +0x16d github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.coalescedHeartbeatsLoop)-fm(0x6a552a0, 0xc420e499e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3878 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063030, 0xc421163e60, 0xc421063020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 662 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x6a552a0, 0xc420e49a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1324 +0x122 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063050, 0xc421163e60, 0xc4203c3420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 663 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc420e49a70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063060, 0xc421163e60, 0xc420ba6a50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 664 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc420e49aa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063070, 0xc421163e60, 0xc420ba6b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 665 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc420e49ad0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063080, 0xc421163e60, 0xc420ba6dc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 682 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc4206c0c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 667 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage/compactor.(*Compactor).Start.func1(0x6a552a0, 0xc420e49b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/compactor/compactor.go:140 +0x1ce github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210630f0, 0xc421163e60, 0xc420e491d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 681 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc421013050, 0xc421012ff0, 0xc4206b1fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc4206c0c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 695 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7f950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203dbed0, 0xc420f747e0, 0xc4203dbec0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 669 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420e49b90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210631c0, 0xc421163e60, 0xc4203c37e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 670 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420e49bc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210631d0, 0xc421163e60, 0xc4203c3820) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 671 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420e49bf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210631e0, 0xc421163e60, 0xc4203c3840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 672 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420e49c20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210631f0, 0xc421163e60, 0xc4203c3860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 673 [chan send, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420e49c50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:594 +0x227 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063200, 0xc421163e60, 0xc4203c3880) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 674 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420e49c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063210, 0xc421163e60, 0xc4203c38e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 675 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420e49cb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063220, 0xc421163e60, 0xc4203c3900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 676 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420e49ce0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063230, 0xc421163e60, 0xc4203c3940) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 677 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420882750, 0x6a552a0, 0xc420e49b30, 0xc421163e60, 0x3ad73798, 0xed2292262, 0x0, 0xc42197fc00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:212 +0x226 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc42197fc00, 0xc421013801) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x87 github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc421013890, 0xc42378cb00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:324 +0x1ba github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x6a552a0, 0xc420e49b30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063240, 0xc421163e60, 0xc4203c3a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 592 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler.func1(0xc420f747e0, 0xc420c70540, 0xc421080230) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:125 +0x10d created by github.com/cockroachdb/cockroach/pkg/security.(*CertificateManager).RegisterSignalHandler /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/security/certificate_manager.go:123 +0xc9 goroutine 593 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/rpc.NewContext.func1(0x6a552a0, 0xc420cee660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:321 +0x51 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9640, 0xc420f747e0, 0xc4203b9120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 614 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc4210f2180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 615 [semacquire]: sync.runtime_notifyListWait(0xc4206b8400, 0xc400000017) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4206b83f0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 io.(*pipe).read(0xc4206b83c0, 0xc42018e000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:47 +0xc6 io.(*PipeReader).Read(0xc42000e258, 0xc42018e000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:130 +0x4c crypto/tls.(*block).readFromUntil(0xc420d87140, 0x73ccb68, 0xc4210ec470, 0x5, 0xc4210ec470, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420edc380, 0x5d2d117, 0xc420edc4a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420edc380, 0xc4212d0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420b3c480, 0xc420d96e38, 0x9, 0x9, 0x9, 0x0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420b3c480, 0xc420d96e38, 0x9, 0x9, 0x9, 0xc4217aed60, 0x400436e, 0xc4217aee0f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420b3c480, 0xc420d96e38, 0x9, 0x9, 0x44f790b, 0xc42310b30c, 0xc42238b280) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420d96e38, 0x9, 0x9, 0x6a34360, 0xc420b3c480, 0x0, 0x0, 0xc42310b300, 0xc4217aee90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420d96e00, 0xc42310b300, 0xc42310b300, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4210f2180, 0xc42086ee40, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4205e8420, 0x6a60d60, 0xc4210f2180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc42037d540, 0xc4205e8420, 0x73ccf20, 0xc4210f2180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 623 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420f0ea40, 0xc421013230, 0xc4210a8fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc4206c0d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 706 [semacquire, 1 minutes]: sync.runtime_notifyListWait(0xc4211b2428, 0xc400000000) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211b2418) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4211b2300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:623 +0xa2 created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:610 +0x44d goroutine 702 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7f8f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204261a0, 0xc420f747e0, 0xc420426010) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 703 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fb00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426560, 0xc420f747e0, 0xc420426420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 704 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fb30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426650, 0xc420f747e0, 0xc420426620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 705 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fb60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204266d0, 0xc420f747e0, 0xc420426690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 722 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fad0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426740, 0xc420f747e0, 0xc420426700) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 723 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fbc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426780, 0xc420f747e0, 0xc420426770) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 724 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fb90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204267a0, 0xc420f747e0, 0xc420426790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 725 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).initRunners.func1(0x6a552a0, 0xc420b7fbf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:86 +0x167 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426810, 0xc420f747e0, 0xc4204267d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 726 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc421e2a700) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1123 +0x68 created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:80 +0x2b8 goroutine 727 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/log.(*loggingT).gcDaemon(0xc421e2a800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/clog.go:1123 +0x68 created by github.com/cockroachdb/cockroach/pkg/util/log.NewSecondaryLogger /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/log/secondary_log.go:80 +0x2b8 goroutine 707 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*LeaseManager).RefreshLeases.func1(0x6a552a0, 0xc420730360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/lease.go:1469 +0x1f3 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378000, 0xc420f747e0, 0xc420dc6540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 708 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/util/netutil.MakeServer.func2(0x6a552a0, 0xc4207303f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:100 +0x64 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203780f0, 0xc420f747e0, 0xc4211d80a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 709 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func2(0x6a552a0, 0xc4210121b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:790 +0x53 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378350, 0xc420f747e0, 0xc4211d81c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 710 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73cc0c0, 0x72, 0xffffffffffffffff) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c32298, 0x72, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c32298, 0xffffffffffffff00, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Accept(0xc420c32280, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:334 +0x1e2 net.(*netFD).accept(0xc420c32280, 0x735ad90, 0x0, 0x1) /Users/andrei/work/src/go/src/net/fd_unix.go:238 +0x42 net.(*TCPListener).accept(0xc4205b0100, 0xc420ae56f8, 0x10, 0x4ae8f17) /Users/andrei/work/src/go/src/net/tcpsock_posix.go:136 +0x2e net.(*TCPListener).Accept(0xc4205b0100, 0x5d2b638, 0xc420f0e140, 0xc4203cc050, 0xc420ae56e0) /Users/andrei/work/src/go/src/net/tcpsock.go:247 +0x49 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc420f0e140, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x95 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func3(0x6a552a0, 0xc4210121e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:802 +0x2f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378360, 0xc420f747e0, 0xc4211d8240) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 711 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4211d81e0, 0xc4200180f8, 0x59e1ac0, 0x6c7b6c0, 0x5c07060) :1 +0x65 net/http.(*Server).Serve(0xc4204f4d00, 0x6a54460, 0xc4211d81e0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/http/server.go:2695 +0x1b2 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func4(0x6a552a0, 0xc421012000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:814 +0x119 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203783f0, 0xc420f747e0, 0xc420dc6720) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 712 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4211d8200, 0xc421012270, 0xc420ae6680, 0x412762a, 0x5b49240) :1 +0x65 crypto/tls.(*listener).Accept(0xc4211d8260, 0xc4200180f8, 0x59e1ac0, 0x6c7b6c0, 0x5c07060) /Users/andrei/work/src/go/src/crypto/tls/tls.go:52 +0x37 net/http.(*Server).Serve(0xc421306410, 0x6a461e0, 0xc4211d8260, 0x0, 0x0) /Users/andrei/work/src/go/src/net/http/server.go:2695 +0x1b2 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func5(0x6a552a0, 0xc421012210) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:821 +0x42 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378400, 0xc420f747e0, 0xc4211d8280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 713 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func6(0x6a552a0, 0xc4210122a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:825 +0x7c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378460, 0xc420f747e0, 0xc420f0e180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 714 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4211d8160, 0x5d2c428, 0xc4205e91e0, 0x6a61a80, 0xc421f9a500) :1 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).Serve(0xc4205e91e0, 0x6a54460, 0xc4211d8160, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:504 +0x1ec github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func7(0x6a552a0, 0xc4210122d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:837 +0x43 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378490, 0xc420f747e0, 0xc4211d82a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 715 [semacquire]: sync.runtime_notifyListWait(0xc4211b31a8, 0xc400002338) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4211b3198) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).syncLoop(0xc4211b3080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:623 +0xa2 created by github.com/cockroachdb/cockroach/pkg/storage/engine.(*RocksDB).open /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/engine/rocksdb.go:610 +0x44d goroutine 716 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).start.func3(0x6a552a0, 0xc420dc68d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:371 +0x5a github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378800, 0xc420f747e0, 0xc4211d8420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 717 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).bootstrap.func1(0x6a552a0, 0xc420dc6930) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1160 +0x46b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378850, 0xc420f747e0, 0xc420378830) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 718 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*Gossip).manage.func1(0x6a552a0, 0xc420dc6ba0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.go:1189 +0x269 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378940, 0xc420f747e0, 0xc420378860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 719 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73cc180, 0x72, 0xffffffffffffffff) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c32098, 0x72, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c32098, 0xffffffffffffff00, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Accept(0xc420c32080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:334 +0x1e2 net.(*netFD).accept(0xc420c32080, 0xc4206eedc0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:238 +0x42 net.(*TCPListener).accept(0xc4205b00e8, 0x4ae900a, 0x405c590, 0xc4210a6e80) /Users/andrei/work/src/go/src/net/tcpsock_posix.go:136 +0x2e net.(*TCPListener).Accept(0xc4205b00e8, 0x5d2b640, 0xc420f0e100, 0x6a61c00, 0xc4206eedc0) /Users/andrei/work/src/go/src/net/tcpsock.go:247 +0x49 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*cMux).Serve(0xc420f0e100, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:124 +0x95 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10.1() /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1011 +0x2f sync.(*Once).Do(0xc420428310, 0xc420bc9f58) /Users/andrei/work/src/go/src/sync/once.go:44 +0xbe github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func10(0x6a552a0, 0xc420dc68a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1010 +0x5b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203789d0, 0xc420f747e0, 0xc4211d8440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 720 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip(0xc421307380, 0x6a552a0, 0xc420dc69f0, 0xc420b29800, 0x6a61240, 0xc420bf8250, 0xc420f747e0, 0xc4204285b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:323 +0x350 github.com/cockroachdb/cockroach/pkg/gossip.(*client).startLocked.func1(0x6a552a0, 0xc420dc69f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:130 +0x3cd github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378ab0, 0xc420f747e0, 0xc420f0e400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 721 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420dc6b10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 738 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc42115a780, 0xc420ae28c0, 0xc4212b1ca0, 0xf, 0xc42126e3c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:581 +0x42c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x6a552a0, 0xc420dc6b70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:512 +0x74 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378c40, 0xc420f747e0, 0xc420dc6b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 739 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420f0e880) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 740 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420e0d180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420e0d180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 622 [IO wait]: internal/poll.runtime_pollWait(0x73cc000, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c32498, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c32498, 0xc421942000, 0x2000, 0x2000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c32480, 0xc421942000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c32480, 0xc421942000, 0x2000, 0x2000, 0x6a, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0178, 0xc421942000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4210123f0, 0x77aa1a0, 0xc4205b0178, 0x5, 0xc4205b0178, 0xc420d4ea60) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420edca80, 0x5d2d117, 0xc420edcba0, 0x40147f7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420edca80, 0xc420ee0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420e1b260, 0xc420d971b8, 0x9, 0x9, 0x0, 0x402e408, 0xc4202cce00) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420e1b260, 0xc420d971b8, 0x9, 0x9, 0x9, 0xc424740600, 0xc420d4ee20, 0x454a13e) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420e1b260, 0xc420d971b8, 0x9, 0x9, 0x5d2c5d8, 0x200050100, 0xc40000211f) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420d971b8, 0x9, 0x9, 0x6a34360, 0xc420e1b260, 0x0, 0x0, 0x0, 0xc4241ca398) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420d97180, 0xc4233d9380, 0xc4233d9380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4206c0d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 815 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420bd4c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 624 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc4206c0d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1323 +0x14b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 683 [IO wait]: internal/poll.runtime_pollWait(0x73cbf40, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c32518, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c32518, 0xc42206a000, 0x2000, 0x2000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c32500, 0xc42206a000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c32500, 0xc42206a000, 0x2000, 0x2000, 0x1f01, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0170, 0xc42206a000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4203e8640, 0xc42206a000, 0x2000, 0x2000, 0x5b473a0, 0xc421012300, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421012330, 0x73ccbf8, 0xc4203e8640, 0x5, 0xc4203e8640, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b78000, 0x5d2d117, 0xc420b78120, 0x4007d18) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b78000, 0xc420bba000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420e1b0e0, 0xc420d96ff8, 0x9, 0x9, 0xc4214500c0, 0x1000f012088c600, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420e1b0e0, 0xc420d96ff8, 0x9, 0x9, 0x9, 0x45472e6, 0xc420c0e4e0, 0xc421914780) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420e1b0e0, 0xc420d96ff8, 0x9, 0x9, 0xc421914780, 0x5a, 0x5a) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc420d96ff8, 0x9, 0x9, 0x6a34360, 0xc420e1b0e0, 0x0, 0x100000000, 0x2b0000005a, 0x5a00000001) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc420d96fc0, 0xc421012f00, 0xc421012f00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4206c0c00, 0xc420d863f0, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc42013e2c0, 0x6a60d60, 0xc4206c0c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc420396040, 0xc42013e2c0, 0x73ccf20, 0xc4206c0c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 754 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func4(0x6a61b40, 0xc4206c0d80, 0xc420e0cf00, 0xc420e187e0, 0xc420b00780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:284 +0x1cd created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:283 +0xaaa goroutine 755 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420ba7e50, 0xc4203c2cf0, 0x5, 0x5, 0x17, 0xc421e53aa8, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420ba7e50, 0xc4203c2cf0, 0x5, 0x5, 0x3, 0xc420029900, 0xc420029970) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420cbe600, 0xc4203c2cf0, 0x5, 0x5, 0x407883e, 0xc420b00878, 0xc4226c5200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc420cbe600, 0xc4203c2cf0, 0x5, 0x5, 0x5, 0xc4206c0d80, 0xc420b00780, 0x5) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc420cbe600, 0xc4203c2cf0, 0x5, 0x5, 0x4, 0xc421e53bd0, 0x4007edc) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420b00780, 0xc4203c2cf0, 0x5, 0x5, 0x4007d18, 0xc420bfa360, 0xc4211c0300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203c2ce0, 0x7fffffff, 0xc421e53ce0, 0x1, 0xc421e53d50, 0x40147f7, 0xc4242a9c20, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203c2ce0, 0x6a54aa0, 0x6cf0d10, 0xc420b00780, 0x0, 0x0, 0x5b26a80, 0xc420b96690, 0x7fffffff, 0xc4242a9c20, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420e187e0, 0x5b26a80, 0xc420b96690, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:442 +0x10c github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipClient).Recv(0xc420bf8250, 0x6a552a0, 0xc420cbe6c0, 0xc420b29800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:191 +0x62 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2.1(0x6a61240, 0xc420bf8250, 0xc421307380, 0x6a552a0, 0xc420cbe6c0, 0xc420b29800, 0xc420bcc770, 0x402dd49) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:311 +0x35 github.com/cockroachdb/cockroach/pkg/gossip.(*client).gossip.func2(0x6a552a0, 0xc420cbe6c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/client.go:319 +0xc0 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf82d0, 0xc420f747e0, 0xc420cf0a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 744 [select]: github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip(0xc4202d8a50, 0x6a612a0, 0xc420378ed0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:185 +0x621 github.com/cockroachdb/cockroach/pkg/gossip._Gossip_Gossip_Handler(0x5be8da0, 0xc4202d8a50, 0x6a5eea0, 0xc4203e9040, 0x4014f58, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:208 +0xb2 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5be8da0, 0xc4202d8a50, 0x6a5eea0, 0xc4203e9040, 0xc4211d8b00, 0x5d27e48, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x103 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc42013e2c0, 0x6a60d60, 0xc4206c0c00, 0xc420421400, 0xc420163440, 0x6c7fb20, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1060 +0x3d6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42013e2c0, 0x6a60d60, 0xc4206c0c00, 0xc420421400, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1147 +0x14c1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203e2010, 0xc42013e2c0, 0x6a60d60, 0xc4206c0c00, 0xc420421400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 745 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420ae2aa0, 0xc4211d8af0, 0x5, 0x5, 0x404ab68, 0xc423d477d0, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420ae2aa0, 0xc4211d8af0, 0x5, 0x5, 0x3, 0xc420028600, 0xc420028670) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420dc6ff0, 0xc4211d8af0, 0x5, 0x5, 0x407883e, 0xc4204214f8, 0xc424039600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc420dc6ff0, 0xc4211d8af0, 0x5, 0x5, 0x5, 0xc4206c0c00, 0xc420421400, 0xc400000005) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc420dc6ff0, 0xc4211d8af0, 0x5, 0x5, 0xc423d478e8, 0x4012f13, 0x9) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420421400, 0xc4211d8af0, 0x5, 0x5, 0x6a43360, 0xc420028688, 0xc420028600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4211d8ae0, 0x7fffffff, 0x4008553, 0xc42126f498, 0xc423d47a78, 0x40147f7, 0xc423c5a600, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4211d8ae0, 0x6a54aa0, 0x6cf0d10, 0xc420421400, 0x0, 0x0, 0x5b42a20, 0xc423ddf770, 0x7fffffff, 0xc423c5a600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc4203e9040, 0x5b42a20, 0xc423ddf770, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:698 +0x12e github.com/cockroachdb/cockroach/pkg/gossip.(*gossipGossipServer).Recv(0xc420378ed0, 0xc423d47bb0, 0x41a6460, 0xc4202d8a98) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/gossip.pb.go:227 +0x62 github.com/cockroachdb/cockroach/pkg/gossip.(Gossip_GossipServer).Recv-fm(0xc4202d8a98, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:148 +0x2f github.com/cockroachdb/cockroach/pkg/gossip.(*server).gossipReceiver(0xc4202d8a50, 0x6a552a0, 0xc420730570, 0xc4205b0180, 0xc420f0ef00, 0xc420da1f58, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:326 +0x474 github.com/cockroachdb/cockroach/pkg/gossip.(*server).Gossip.func3.1(0x6a552a0, 0xc420730570) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/gossip/server.go:148 +0x96 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378f10, 0xc4204ba360, 0xc420f0ef40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1788 [IO wait]: internal/poll.runtime_pollWait(0x7724840, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96098, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96098, 0xc4220c4c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96080, 0xc4220c4c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96080, 0xc4220c4c00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e400, 0xc4220c4c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4221c8140, 0xc4220c4c00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b23bf0, 0x73ccbf8, 0xc4221c8140, 0x5, 0xc4221c8140, 0xa9) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42138f500, 0x5d2d117, 0xc42138f620, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42138f500, 0xc421b6c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422d3b020, 0xc421b6c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc9aa0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc9aa0, 0xbe9de8a328d28fc0, 0x7c92be0bd, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc9bd0, 0x6a46020, 0xc422fc9aa0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc9a40, 0x6a551e0, 0xc42121b140, 0xc423200d60, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4201caac0, 0x6a611e0, 0xc42138f500, 0x0, 0x0, 0xc4216bb022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4201caac0, 0x6a611e0, 0xc42138f500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222be5d0, 0x6a61a80, 0xc4221c8140, 0xc4203a3b00, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4221c8140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc4221c8140, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1217 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc4219fc240, 0xc4219fc1b0, 0xc4213c6fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc421624600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 114955 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422750280, 0x45563f3, 0xc4245811e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422750280, 0x5d2c558, 0xc424581420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4219f7950, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4219f7950, 0x5c6d3d9, 0x21, 0x5c13340, 0xc420c754b8, 0x5b6ec40, 0xc4238eef00, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42397d4c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc420c754b8, 0x5b6ec40, 0xc4238eef00, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc42397d4c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc420c754b8, 0x5b6ec40, 0xc4238eef00, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42397d4c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc420c754b8, 0x5b6ec40, 0xc4238eef00, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f9748, 0x6a551e0, 0xc42397d4c0, 0xc420c754b8, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x73d4c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423747e30, 0xc420c754a0, 0x6a551e0, 0xc42397d4c0, 0xc400000008, 0xc424dcffc0, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423747e30, 0x6a551e0, 0xc42397d4c0, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423747e30, 0x6a551e0, 0xc42397d4c0, 0xc420c753f0, 0xc4238eeea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 920 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc420518000, 0x2, 0xc4211ed080, 0xc420b4e100, 0x6a61900, 0xc42043f5b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:517 +0x54a github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc420518000, 0x2, 0x6a552a0, 0xc420c5bb00, 0x2, 0x0, 0xc4213ce5a0, 0xc4211ed080, 0xc420b4e100, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x2e2 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc420518000, 0x6a552a0, 0xc420c5bb00, 0x2, 0xc4211ed080, 0xc420b4e100) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:463 +0xd0 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func1.1(0x6a552a0, 0xc420c5bb00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:591 +0x60 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378630, 0xc4204ba360, 0xc420dc79b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 915 [IO wait]: internal/poll.runtime_pollWait(0x73cbd00, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb718, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb718, 0xc4239be000, 0x4000, 0x4000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb700, 0xc4239be000, 0x4000, 0x4000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb700, 0xc4239be000, 0x4000, 0x4000, 0x6a, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e690, 0xc4239be000, 0x4000, 0x4000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc421065bc0, 0x77aa1a0, 0xc42000e690, 0x5, 0xc42000e690, 0xc42201ba60) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4210d7c00, 0x5d2d117, 0xc4210d7d20, 0x40147f7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4210d7c00, 0xc421362000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4211ec120, 0xc4204a4b98, 0x9, 0x9, 0x0, 0x402e408, 0xc421945880) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4211ec120, 0xc4204a4b98, 0x9, 0x9, 0x9, 0xc4247406c0, 0xc42201be20, 0x454a13e) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4211ec120, 0xc4204a4b98, 0x9, 0x9, 0x5d2c5d8, 0x200050100, 0xc400006fa1) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4204a4b98, 0x9, 0x9, 0x6a34360, 0xc4211ec120, 0x0, 0x0, 0x0, 0xc4241ca3a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4204a4b60, 0xc4233d9e60, 0xc4233d9e60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc421200180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 906 [IO wait]: internal/poll.runtime_pollWait(0x73cbdc0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420462c98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420462c98, 0xc424776000, 0x8000, 0x8000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420462c80, 0xc424776000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420462c80, 0xc424776000, 0x8000, 0x8000, 0x7f01, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e4e8, 0xc424776000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4208895e0, 0xc424776000, 0x8000, 0x8000, 0x5b473a0, 0xc420db8e00, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420db8ea0, 0x73ccbf8, 0xc4208895e0, 0x5, 0xc4208895e0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4210d6000, 0x5d2d117, 0xc4210d6120, 0x4007d18) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4210d6000, 0xc42210c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc42210a120, 0xc4210c2658, 0x9, 0x9, 0xc4238bfa00, 0x1000f01206c1e00, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc42210a120, 0xc4210c2658, 0x9, 0x9, 0x9, 0x45472e6, 0xc420c17950, 0xc421b20e40) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc42210a120, 0xc4210c2658, 0x9, 0x9, 0xc421b20e40, 0x59, 0x59) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4210c2658, 0x9, 0x9, 0x6a34360, 0xc42210a120, 0x0, 0x100000000, 0x500000059, 0x5900000001) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4210c2620, 0xc420db9d10, 0xc420db9d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421a3f500, 0xc420c008a0, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4205e91e0, 0x6a60d60, 0xc421a3f500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc420383460, 0xc4205e91e0, 0x73ccf20, 0xc421a3f500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 898 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc42109ef00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc42109ef00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 890 [IO wait]: internal/poll.runtime_pollWait(0x73cbe80, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb318, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb318, 0xc422084c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb300, 0xc422084c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb300, 0xc422084c00, 0x400, 0x400, 0x6a, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee518, 0xc422084c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420db8f60, 0x77aa1a0, 0xc4206ee518, 0x5, 0xc4206ee518, 0xc422131a60) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420f85880, 0x5d2d117, 0xc420f859a0, 0x40147f7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420f85880, 0xc42211c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc42210a240, 0xc4210c2818, 0x9, 0x9, 0x0, 0x402e408, 0xc4202cce00) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc42210a240, 0xc4210c2818, 0x9, 0x9, 0x9, 0xc424740670, 0xc422131e20, 0x454a13e) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc42210a240, 0xc4210c2818, 0x9, 0x9, 0x5d2c5d8, 0x200050100, 0xc4000000cf) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4210c2818, 0x9, 0x9, 0x6a34360, 0xc42210a240, 0x0, 0x0, 0x0, 0xc4241ca3a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4210c27e0, 0xc4233d9a40, 0xc4233d9a40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc421a3f680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 878 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc4220e4000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 868 [chan receive, 1 minutes]: database/sql.(*DB).connectionOpener(0xc421050280) /Users/andrei/work/src/go/src/database/sql/sql.go:871 +0x53 created by database/sql.Open /Users/andrei/work/src/go/src/database/sql/sql.go:609 +0x1ee goroutine 879 [semacquire]: sync.runtime_notifyListWait(0xc4206b99c0, 0xc400000016) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4206b99b0) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 io.(*pipe).read(0xc4206b9980, 0xc420711400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:47 +0xc6 io.(*PipeReader).Read(0xc4205b01a8, 0xc420711400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:130 +0x4c crypto/tls.(*block).readFromUntil(0xc420dc7680, 0x73ccb68, 0xc420378ff0, 0x5, 0xc420378ff0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206f9c00, 0x5d2d117, 0xc4206f9d20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206f9c00, 0xc4220f0000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420fb2f60, 0xc4204a5c38, 0x9, 0x9, 0x9, 0x0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420fb2f60, 0xc4204a5c38, 0x9, 0x9, 0x9, 0xc420e8ed60, 0x400436e, 0xc420e8ee0f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420fb2f60, 0xc4204a5c38, 0x9, 0x9, 0x44f790b, 0xc424b9358c, 0xc422bdecb0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4204a5c38, 0x9, 0x9, 0x6a34360, 0xc420fb2f60, 0x0, 0x0, 0xc424b93580, 0xc420e8ee90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4204a5c00, 0xc424b93580, 0xc424b93580, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4220e4000, 0xc420c16a50, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4205e91e0, 0x6a60d60, 0xc4220e4000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc4203c87e0, 0xc4205e91e0, 0x73ccf20, 0xc4220e4000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 819 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func1(0x6a552a0, 0xc4211dad20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:156 +0x49 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ecf70, 0xc420f747e0, 0xc421095a00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 817 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420e00a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 904 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc420db9ec0, 0xc420db9e60, 0xc420e8afb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc421a3f500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 746 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc420428314, 0xc420f0f300) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc420428310) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee sync.(*Once).Do(0xc420428310, 0xc420bc6758) /Users/andrei/work/src/go/src/sync/once.go:40 +0x44 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func11(0x6a552a0, 0xc420dc75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1031 +0x5b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420378f40, 0xc420f747e0, 0xc4211d8c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 747 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func12(0x6a552a0, 0xc420dc7620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1073 +0x7c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420379010, 0xc420f747e0, 0xc420dc7380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 752 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420f0f500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 749 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc420dc7440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 750 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func15(0x6a552a0, 0xc420dc75c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1103 +0x53 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420379100, 0xc420f747e0, 0xc4211d8fc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 880 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420f0f640, 0xc42086f680, 0xc420cccfb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc420e61800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 753 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc420e0d680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc420e0d680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 771 [semacquire]: sync.runtime_notifyListWait(0xc4206b9a80, 0xc400000015) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc4206b9a70) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 io.(*pipe).read(0xc4206b9a40, 0xc420534400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:47 +0xc6 io.(*PipeReader).Read(0xc4205b01b8, 0xc420534400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/pipe.go:130 +0x4c crypto/tls.(*block).readFromUntil(0xc420dc7860, 0x73ccb68, 0xc420379000, 0x5, 0xc420379000, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420065180, 0x5d2d117, 0xc4200652a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420065180, 0xc421b72000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420d953e0, 0xc42100ac78, 0x9, 0x9, 0xc420e61200, 0x4, 0xc4206aed88) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420d953e0, 0xc42100ac78, 0x9, 0x9, 0x9, 0xc4206aee18, 0x3, 0xc4206aee8f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420d953e0, 0xc42100ac78, 0x9, 0x9, 0x44f790b, 0xc424b9356c, 0xc420d95498) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42100ac78, 0x9, 0x9, 0x6a34360, 0xc420d953e0, 0x0, 0x0, 0xc42086f688, 0xc4206aeed0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42100ac40, 0xc424b93560, 0xc424b93560, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc420e61800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 736 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startComputePeriodicMetrics.func1(0x6a552a0, 0xc420cbec90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:736 +0x14f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db4b0, 0xc420f747e0, 0xc420eaaa80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 877 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc420c169f0, 0xc420c16990, 0xc422090fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc4220e4000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 907 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420e01880) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 688 [chan receive, 1 minutes]: github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.muxListener.Accept(...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:184 github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*muxListener).Accept(0xc4211d8140, 0x5d2b520, 0xc420f747e0, 0x6a552a0, 0xc420f6d500) :1 +0x65 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith(0xc4205b0080, 0x6a552a0, 0xc420f6d500, 0xc420f747e0, 0x6a54460, 0xc4211d8140, 0xc420f6d530, 0xc420f6d500, 0xc4212736d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:119 +0x50 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17(0x6a552a0, 0xc420f6d500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1277 +0x1f5 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ecd80, 0xc420f747e0, 0xc42121b500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 737 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startGossip.func1(0x6a552a0, 0xc420cbeff0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:698 +0x2dd github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db4c0, 0xc420f747e0, 0xc420eaaaa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 786 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/server.(*Server).refreshSettings.func2(0x6a552a0, 0xc420cbef90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/settingsworker.go:112 +0x2db github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db4f0, 0xc420f747e0, 0xc420eaad00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 787 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Server).startSampleEnvironment.func1(0x6a552a0, 0xc420cbf0b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1465 +0x16c github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db560, 0xc420f747e0, 0xc420eaad40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 788 [select]: github.com/cockroachdb/cockroach/pkg/ts.(*poller).start.func1(0x6a552a0, 0xc420b222a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/ts/db.go:126 +0x156 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db590, 0xc420f747e0, 0xc4203db570) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 789 [select]: github.com/cockroachdb/cockroach/pkg/server.(*Node).startWriteSummaries.func1(0x6a552a0, 0xc420cbeea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:773 +0x1f6 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db5a0, 0xc420f747e0, 0xc420eaad80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 790 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*SchemaChangeManager).Start.func1(0x6a552a0, 0xc420cbef60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/schema_changer.go:1031 +0xa6b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db5b0, 0xc420f747e0, 0xc420eaadc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 791 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*Executor).Start.func1(0x6a552a0, 0xc420cbee40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/executor.go:397 +0x129 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db600, 0xc420f747e0, 0xc420eaade0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 816 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420420f00, 0xc420ef45f0, 0xc420ce9fd0, 0xf, 0xc420f25140, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:581 +0x42c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x6a552a0, 0xc420bd4c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:512 +0x74 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9a60, 0xc4204ba360, 0xc420bd4c30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 908 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc4221a6000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc4221a6000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 794 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*flowScheduler).Start.func1(0x6a552a0, 0xc420cbf140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow_scheduler.go:125 +0x17b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db6e0, 0xc420f747e0, 0xc4203db6d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 795 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/sql.(*Server).Start.func1(0x6a552a0, 0xc420ea9c20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:286 +0x133 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db710, 0xc420f747e0, 0xc420eaaec0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 796 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*NodeLiveness).StartHeartbeat.func1(0x6a552a0, 0xc420ea9c50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/node_liveness.go:446 +0x2a9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db860, 0xc420f747e0, 0xc420f92370) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 797 [select]: github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func1(0x6a552a0, 0xc420b26390) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:205 +0x165 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db870, 0xc420f747e0, 0xc420ea9980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 798 [select]: github.com/cockroachdb/cockroach/pkg/sql/jobs.(*Registry).Start.func2(0x6a552a0, 0xc420b263c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/jobs/registry.go:216 +0x13b github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203db880, 0xc420f747e0, 0xc420ea99b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 905 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc421a3f500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 820 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004061) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dad50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dad50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ecf90, 0xc420f747e0, 0xc4210ecf80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 821 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000406b) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dad80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dad80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ecfb0, 0xc420f747e0, 0xc4210ecfa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 822 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004075) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dadb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dadb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ecfd0, 0xc420f747e0, 0xc4210ecfc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 823 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004078) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dade0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dade0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ecff0, 0xc420f747e0, 0xc4210ecfe0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 824 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004068) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dae10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dae10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed010, 0xc420f747e0, 0xc4210ed000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 825 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004072) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dae40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dae40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed030, 0xc420f747e0, 0xc4210ed020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 826 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000407b) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dae70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dae70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed050, 0xc420f747e0, 0xc4210ed040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 827 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000407e) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211daea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211daea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed070, 0xc420f747e0, 0xc4210ed060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 828 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004069) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211daed0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211daed0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed090, 0xc420f747e0, 0xc4210ed080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 829 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004070) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211daf00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211daf00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed0b0, 0xc420f747e0, 0xc4210ed0a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 830 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000405f) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211daf30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211daf30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed0d0, 0xc420f747e0, 0xc4210ed0c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 831 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004064) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211daf60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211daf60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed0f0, 0xc420f747e0, 0xc4210ed0e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 832 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004060) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211daf90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211daf90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed110, 0xc420f747e0, 0xc4210ed100) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 833 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004079) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dafc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dafc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed130, 0xc420f747e0, 0xc4210ed120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 834 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000406d) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211daff0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211daff0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed150, 0xc420f747e0, 0xc4210ed140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 835 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004062) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211db020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211db020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed170, 0xc420f747e0, 0xc4210ed160) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 836 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000407c) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211db050) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211db050) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed190, 0xc420f747e0, 0xc4210ed180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 837 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000406c) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211db080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211db080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed1b0, 0xc420f747e0, 0xc4210ed1a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 838 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000407a) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211db0e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211db0e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed1d0, 0xc420f747e0, 0xc4210ed1c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 839 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004063) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc4211dacf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc4211dacf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed1f0, 0xc420f747e0, 0xc4210ed1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 840 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004065) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e92e70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e92e70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed210, 0xc420f747e0, 0xc4210ed200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 841 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004077) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e92ea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e92ea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed230, 0xc420f747e0, 0xc4210ed220) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 842 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000406f) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e92f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e92f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed250, 0xc420f747e0, 0xc4210ed240) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 843 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000407d) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e92f30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e92f30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed270, 0xc420f747e0, 0xc4210ed260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 844 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004076) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e92f60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e92f60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed290, 0xc420f747e0, 0xc4210ed280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 845 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004071) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e92f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e92f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed2b0, 0xc420f747e0, 0xc4210ed2a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 846 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004073) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e92fc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e92fc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed2d0, 0xc420f747e0, 0xc4210ed2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 847 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000406e) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e92ff0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e92ff0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed2f0, 0xc420f747e0, 0xc4210ed2e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 848 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc40000406a) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e93020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e93020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed310, 0xc420f747e0, 0xc4210ed300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 849 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004074) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e93050) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e93050) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed330, 0xc420f747e0, 0xc4210ed320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 850 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004066) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e93080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e93080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed350, 0xc420f747e0, 0xc4210ed340) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 851 [semacquire]: sync.runtime_notifyListWait(0xc420747290, 0xc400004067) /Users/andrei/work/src/go/src/runtime/sema.go:507 +0x110 sync.(*Cond).Wait(0xc420747280) /Users/andrei/work/src/go/src/sync/cond.go:56 +0x80 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).worker(0xc420f47560, 0x6a552a0, 0xc420e930b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:197 +0x7c github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Start.func2(0x6a552a0, 0xc420e930b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:166 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed370, 0xc420f747e0, 0xc4210ed360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 852 [semacquire, 1 minutes]: sync.runtime_Semacquire(0xc420f475e4) /Users/andrei/work/src/go/src/runtime/sema.go:56 +0x39 sync.(*WaitGroup).Wait(0xc420f475d8) /Users/andrei/work/src/go/src/sync/waitgroup.go:131 +0x72 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait(0xc420f47560, 0x6a552a0, 0xc420e930e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scheduler.go:172 +0x31 github.com/cockroachdb/cockroach/pkg/storage.(*raftScheduler).Wait-fm(0x6a552a0, 0xc420e930e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3875 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed390, 0xc420f747e0, 0xc4210ed380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 853 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).raftTickLoop(0xc420f41000, 0x6a552a0, 0xc420e93110) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3891 +0x205 github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.raftTickLoop)-fm(0x6a552a0, 0xc420e93110) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3877 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed3c0, 0xc420f747e0, 0xc4210ed3a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 854 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).coalescedHeartbeatsLoop(0xc420f41000, 0x6a552a0, 0xc420e93140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3932 +0x16d github.com/cockroachdb/cockroach/pkg/storage.(*Store).(github.com/cockroachdb/cockroach/pkg/storage.coalescedHeartbeatsLoop)-fm(0x6a552a0, 0xc420e93140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:3878 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed3e0, 0xc420f747e0, 0xc4210ed3d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 855 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).Start.func2(0x6a552a0, 0xc420e93230) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1324 +0x122 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed400, 0xc420f747e0, 0xc421095a20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 856 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc420e93290) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed410, 0xc420f747e0, 0xc421259810) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 857 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc420e932c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed420, 0xc420f747e0, 0xc421259860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 858 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Store).startGossip.func4(0x6a552a0, 0xc420e932f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:1458 +0x3e9 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed430, 0xc420f747e0, 0xc4212598b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 892 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc421a3f680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1323 +0x14b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 860 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage/compactor.(*Compactor).Start.func1(0x6a552a0, 0xc420e93380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/compactor/compactor.go:140 +0x1ce github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed4e0, 0xc420f747e0, 0xc420e92840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 891 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420e00b40, 0xc420c4a060, 0xc4206affb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc421a3f680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 862 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420bd44e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed6a0, 0xc420f747e0, 0xc420364640) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 863 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420bd4510) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed6b0, 0xc420f747e0, 0xc420364660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 864 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420bd4540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed6c0, 0xc420f747e0, 0xc420364680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 865 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420bd4570) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed6d0, 0xc420f747e0, 0xc4203646c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 882 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420bd45a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed6e0, 0xc420f747e0, 0xc4203646e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 883 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420bd44b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed6f0, 0xc420f747e0, 0xc420364700) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 884 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420cef1a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed700, 0xc420f747e0, 0xc4203648e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 885 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1(0x6a552a0, 0xc420bd4990) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:575 +0x1b2 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed710, 0xc420f747e0, 0xc420364900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 886 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).waitAndProcess(0xc420cecd00, 0x6a552a0, 0xc420e93350, 0xc420f747e0, 0x9464868, 0xed2292260, 0x0, 0xc42197f180, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:212 +0x226 github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1.1(0xc42197f180, 0xc420e48a01) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:274 +0x87 github.com/cockroachdb/cockroach/pkg/storage.(*storeReplicaVisitor).Visit(0xc420e48a80, 0xc424919740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:324 +0x1ba github.com/cockroachdb/cockroach/pkg/storage.(*replicaScanner).scanLoop.func1(0x6a552a0, 0xc420e93350) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/scanner.go:272 +0x1dc github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210ed720, 0xc420f747e0, 0xc420364920) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 916 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420e01980, 0xc420dc6720, 0xc420dc0fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc421200180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 896 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc4210659b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 940 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc4210e25a0, 0xc4210e2540, 0xc421134fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc4212da000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 897 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420420f00, 0xc420bd00f0, 0xc420412130, 0xf, 0xc4221a00c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:581 +0x42c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x6a552a0, 0xc421065a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:512 +0x74 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4211a26b0, 0xc4204ba360, 0xc4210659e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 946 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc420518000, 0x6a61960, 0xc4212c8810, 0x6a42020, 0xc420518000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:359 +0x215 github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x5c01800, 0xc420518000, 0x6a5eea0, 0xc420888be0, 0x4014f58, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:375 +0xb2 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5c01800, 0xc420518000, 0x6a5eea0, 0xc420888be0, 0xc4203e4260, 0x5d2b180, 0x5, 0x5) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x103 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc42013e2c0, 0x6a60d60, 0xc4206c0780, 0xc420b01180, 0xc4204e0300, 0x6c88b00, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1060 +0x3d6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42013e2c0, 0x6a60d60, 0xc4206c0780, 0xc420b01180, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1147 +0x14c1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203747b0, 0xc42013e2c0, 0x6a60d60, 0xc4206c0780, 0xc420b01180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 930 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc420518000, 0x3, 0xc420fb3c20, 0xc4212f8500, 0x6a61900, 0xc421063390, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:517 +0x54a github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc420518000, 0x3, 0x6a552a0, 0xc420c175c0, 0x1, 0x0, 0xc420889180, 0xc420fb3c20, 0xc4212f8500, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x2e2 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc420518000, 0x6a552a0, 0xc420c175c0, 0x3, 0xc420fb3c20, 0xc4212f8500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:463 +0xd0 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func1.1(0x6a552a0, 0xc420c175c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:591 +0x60 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063350, 0xc4204ba360, 0xc420c17500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 931 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func4(0x6a61b40, 0xc421a3f680, 0xc42109ec80, 0xc420b69c20, 0xc422096f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:284 +0x1cd created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:283 +0xaaa goroutine 932 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422096f00, 0x4007501, 0xc420fb3f80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc422096f00, 0xc420fb3f80, 0x30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:271 +0x2b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420b69c20, 0x5b334a0, 0xc420c17830, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:428 +0x81a github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc421063390, 0xc420c17800, 0xc420c17800, 0x5a0a5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:326 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x6a61900, 0xc421063390, 0xc4212f8500, 0xc420518000, 0x6a552a0, 0xc420c17800, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x35 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x6a552a0, 0xc420c17800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x66 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210633b0, 0xc4204ba360, 0xc420c177a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 933 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc420518f00, 0x6a61960, 0xc421063410, 0x6a42020, 0xc420518f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:359 +0x215 github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x5c01800, 0xc420518f00, 0x6a5eea0, 0xc4221c8280, 0x4014f58, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:375 +0xb2 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5c01800, 0xc420518f00, 0x6a5eea0, 0xc4221c8280, 0xc4203c8f40, 0x5d2b180, 0x5, 0x5) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x103 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4205e91e0, 0x6a60d60, 0xc421a3f500, 0xc422097180, 0xc420e89500, 0x6c88b00, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1060 +0x3d6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e91e0, 0x6a60d60, 0xc421a3f500, 0xc422097180, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1147 +0x14c1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203bf200, 0xc4205e91e0, 0x6a60d60, 0xc421a3f500, 0xc422097180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 934 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420edecd0, 0xc4203c8f30, 0x5, 0x5, 0x0, 0xc424bddae8, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420edecd0, 0xc4203c8f30, 0x5, 0x5, 0x3, 0xc420028600, 0xc420028670) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420c17a40, 0xc4203c8f30, 0x5, 0x5, 0x407883e, 0xc422097278, 0xc422ef51c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc420c17a40, 0xc4203c8f30, 0x5, 0x5, 0x5, 0xc421a3f500, 0xc422097180, 0x5) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc420c17a40, 0xc4203c8f30, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc422097180, 0xc4203c8f30, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203c8f20, 0x7fffffff, 0x0, 0x0, 0xc424bddd90, 0x40147f7, 0xc4231d6660, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203c8f20, 0x6a54aa0, 0x6cf0d10, 0xc422097180, 0x0, 0x0, 0x5b333c0, 0xc4223a16a0, 0x7fffffff, 0xc4231d6660, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc4221c8280, 0x5b333c0, 0xc4223a16a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:698 +0x12e github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc421063410, 0x6a552a0, 0xc420c17ad0, 0xc422bf21a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:394 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x6a61960, 0xc421063410, 0xc420518f00, 0x6a552a0, 0xc420c17ad0, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x83 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x6a552a0, 0xc420c17ad0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x5d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063430, 0xc420f747e0, 0xc420c17aa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 935 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc420518f00, 0xc400000001, 0xc4221fc6c0, 0xc4212f8580, 0x6a61900, 0xc421063520, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:517 +0x54a github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc420518f00, 0x1, 0x6a552a0, 0xc420c17f50, 0x0, 0x0, 0xc4221c8320, 0xc4221fc6c0, 0xc4212f8580, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x2e2 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc420518f00, 0x6a552a0, 0xc420c17f50, 0x1, 0xc4221fc6c0, 0xc4212f8580) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:463 +0xd0 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func1.1(0x6a552a0, 0xc420c17f50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:591 +0x60 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210634e0, 0xc420f747e0, 0xc420c17f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 936 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func4(0x6a61b40, 0xc4206c0d80, 0xc420e0cf00, 0xc420b69e60, 0xc422097400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:284 +0x1cd created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:283 +0xaaa goroutine 937 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422097400, 0x1, 0xc4221fca20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc422097400, 0xc4221fca20, 0x30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:271 +0x2b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420b69e60, 0x5b334a0, 0xc420c0e300, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:428 +0x81a github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc421063520, 0xc420c0e2d0, 0xc420c0e2d0, 0x5a0a5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:326 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x6a61900, 0xc421063520, 0xc4212f8580, 0xc420518f00, 0x6a552a0, 0xc420c0e2d0, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x35 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x6a552a0, 0xc420c0e2d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x66 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc421063560, 0xc420f747e0, 0xc420c0e270) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 938 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc420518000, 0x6a61960, 0xc4210635c0, 0x6a42020, 0xc420518000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:359 +0x215 github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x5c01800, 0xc420518000, 0x6a5eea0, 0xc4221c8460, 0x4014f58, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:375 +0xb2 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5c01800, 0xc420518000, 0x6a5eea0, 0xc4221c8460, 0xc4203c93c0, 0x5d2b180, 0x5, 0x5) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x103 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc42013e2c0, 0x6a60d60, 0xc4206c0c00, 0xc422097680, 0xc4204e0300, 0x6c88b00, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1060 +0x3d6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc42013e2c0, 0x6a60d60, 0xc4206c0c00, 0xc422097680, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1147 +0x14c1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203e2010, 0xc42013e2c0, 0x6a60d60, 0xc4206c0c00, 0xc422097680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 939 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420edefa0, 0xc4203c93b0, 0x5, 0x5, 0x0, 0xc422218ae8, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420edefa0, 0xc4203c93b0, 0x5, 0x5, 0x3, 0xc420026000, 0xc420026070) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420c0e5a0, 0xc4203c93b0, 0x5, 0x5, 0x407883e, 0xc422097778, 0xc422ef5300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc420c0e5a0, 0xc4203c93b0, 0x5, 0x5, 0x5, 0xc4206c0c00, 0xc422097680, 0x5) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc420c0e5a0, 0xc4203c93b0, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc422097680, 0xc4203c93b0, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203c93a0, 0x7fffffff, 0x0, 0x0, 0xc422218d90, 0x40147f7, 0xc4238c1080, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203c93a0, 0x6a54aa0, 0x6cf0d10, 0xc422097680, 0x0, 0x0, 0x5b333c0, 0xc4222eece0, 0x7fffffff, 0xc4238c1080, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc4221c8460, 0x5b333c0, 0xc4222eece0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:698 +0x12e github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4210635c0, 0x6a552a0, 0xc420c0e630, 0xc42387d520) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:394 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x6a61960, 0xc4210635c0, 0xc420518000, 0x6a552a0, 0xc420c0e630, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x83 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x6a552a0, 0xc420c0e630) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x5d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4210635e0, 0xc4204ba360, 0xc420c0e600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 917 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc421200180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1323 +0x14b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 941 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc4212da000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 942 [IO wait]: internal/poll.runtime_pollWait(0x73cbc40, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c32a98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c32a98, 0xc4225c8000, 0x8000, 0x8000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c32a80, 0xc4225c8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c32a80, 0xc4225c8000, 0x8000, 0x8000, 0x1, 0xc421136898, 0x404b58a) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0468, 0xc4225c8000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015eb40, 0xc4225c8000, 0x8000, 0x8000, 0x5b473a0, 0xc420c00a00, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420c00ab0, 0x73ccbf8, 0xc42015eb40, 0x5, 0xc42015eb40, 0xf) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42138e700, 0x5d2d117, 0xc42138e820, 0x30) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42138e700, 0xc4220bc000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4200cdf80, 0xc42100a738, 0x9, 0x9, 0x1, 0xc42380bfb0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4200cdf80, 0xc42100a738, 0x9, 0x9, 0x9, 0x45472e6, 0xc42380bd70, 0x0) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4200cdf80, 0xc42100a738, 0x9, 0x9, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42100a738, 0x9, 0x9, 0x6a34360, 0xc4200cdf80, 0x0, 0x1000100000000, 0x6fa100000060, 0x6000010001) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42100a700, 0xc4210e2450, 0xc4210e2450, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4212da000, 0xc4210e2630, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4205e8420, 0x6a60d60, 0xc4212da000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc42000d080, 0xc4205e8420, 0x73ccf20, 0xc4212da000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 945 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc4206a42d0, 0x6a61960, 0xc42043fcb0, 0x6a42020, 0xc4206a42d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:359 +0x215 github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x5c01800, 0xc4206a42d0, 0x6a5eea0, 0xc420f45e00, 0x4014f58, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:375 +0xb2 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5c01800, 0xc4206a42d0, 0x6a5eea0, 0xc420f45e00, 0xc4203ec200, 0x5d2b180, 0xc4210307e0, 0xc421270cb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x103 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc420b00b40, 0xc420ea81e0, 0x6c88b00, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1060 +0x3d6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc420b00b40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1147 +0x14c1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc420b00b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 2010 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc4226becf0, 0xc4226bec90, 0xc422b91fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc422eea780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 115248 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42130e0f0, 0xc422528b60, 0xc42390b3e0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42130e0f0, 0x15179d570d92c998, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4225d6200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42130e0f0, 0x15179d570d92c998, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4225d6200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42130e0f0, 0x15179d570d92c998, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4225d6200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42130e090, 0x15179d570d92c998, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4225d6200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420bd5f50, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4225d6200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420bd5f50, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420bd5e90, 0x5c4d11c, 0x10, 0xc424b892c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420bd5e90, 0xc422528af0, 0xc420bd5e90, 0xc4267ed568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420bd5e90, 0xc422528af0, 0xc420b53500, 0xc423f80940, 0xc422e30900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4219f6cf0, 0x5c13340, 0xc422528af0, 0x5c13340, 0xc422528af0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4219f6cf0, 0x5c13340, 0xc422528af0, 0xc421c229e0, 0xc421c22a00, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4219f6cf0, 0x5c13340, 0xc422528af0, 0xc421c229e0, 0xc421c22a00, 0x6, 0xc4267ed968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4219f6cf0, 0xc422528930, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc421920c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424902280, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424902280, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424902280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 1204 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc42130c090) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 115092 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc421117290, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421117290, 0x15179d570a1a0240, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421117290, 0x15179d570a1a0240, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421117290, 0x15179d570a1a0240, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421117260, 0x15179d570a1a0240, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421117200, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421117200, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421117200, 0x5c4d11c, 0x10, 0xc424bbb9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421117200, 0xc423f5e178, 0xc421117200, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421117200, 0xc423f5e178, 0x1f, 0xc420a82630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f59420, 0xc423f5e160, 0x6a551e0, 0xc4241d3100, 0xc400000008, 0xc423aa0b40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f59420, 0x6a551e0, 0xc4241d3100, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f59420, 0x6a551e0, 0xc4241d3100, 0xc423f5e0b0, 0xc422c3c360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 780 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func4(0x6a61b40, 0xc421200180, 0xc422188780, 0xc420c79680, 0xc420e54280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:284 +0x1cd created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:283 +0xaaa goroutine 781 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc420e54280, 0x6a6bc01, 0xc4211c1b00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc420e54280, 0xc4211c1b00, 0x30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:271 +0x2b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420c79680, 0x5b334a0, 0xc420c5be60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:428 +0x81a github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc42043f5b0, 0xc420c5be30, 0xc420c5be30, 0x5a0a5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:326 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x6a61900, 0xc42043f5b0, 0xc420b4e100, 0xc420518000, 0x6a552a0, 0xc420c5be30, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x35 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x6a552a0, 0xc420c5be30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x66 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043f6e0, 0xc4204ba360, 0xc420c5bdd0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 782 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420bd1130, 0xc4203ec190, 0x5, 0x5, 0x0, 0xc425591ae8, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420bd1130, 0xc4203ec190, 0x5, 0x5, 0x3, 0xc420028600, 0xc420028670) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc4210e3830, 0xc4203ec190, 0x5, 0x5, 0x407883e, 0xc420b00c38, 0xc4226c5200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc4210e3830, 0xc4203ec190, 0x5, 0x5, 0x5, 0xc4212da000, 0xc420b00b40, 0x5) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc4210e3830, 0xc4203ec190, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420b00b40, 0xc4203ec190, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203ec180, 0x7fffffff, 0x0, 0x0, 0xc425591d90, 0x40147f7, 0xc424200660, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203ec180, 0x6a54aa0, 0x6cf0d10, 0xc420b00b40, 0x0, 0x0, 0x5b333c0, 0xc4223a0a60, 0x7fffffff, 0xc424200660, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420f45e00, 0x5b333c0, 0xc4223a0a60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:698 +0x12e github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc42043fcb0, 0x6a552a0, 0xc420c5bef0, 0xc42282b6c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:394 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x6a61960, 0xc42043fcb0, 0xc4206a42d0, 0x6a552a0, 0xc420c5bef0, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x83 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x6a552a0, 0xc420c5bef0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x5d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043fd40, 0xc421163e60, 0xc420c5bec0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 783 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc4206a42d0, 0xc400000001, 0xc421142420, 0xc42001d000, 0x6a61900, 0xc42048c1c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:517 +0x54a github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc4206a42d0, 0x1, 0x6a552a0, 0xc420c163f0, 0x0, 0x0, 0xc420f45ea0, 0xc421142420, 0xc42001d000, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x2e2 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc4206a42d0, 0x6a552a0, 0xc420c163f0, 0x1, 0xc421142420, 0xc42001d000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:463 +0xd0 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func1.1(0x6a552a0, 0xc420c163f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:591 +0x60 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c120, 0xc421163e60, 0xc420c163c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 784 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func4(0x6a61b40, 0xc4206c0a80, 0xc420e0c500, 0xc420c79b00, 0xc420e54500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:284 +0x1cd created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:283 +0xaaa goroutine 785 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc420e54500, 0x1, 0xc420c60060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc420e54500, 0xc420c60060, 0x30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:271 +0x2b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc420c79b00, 0x5b334a0, 0xc420c4a3f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:428 +0x81a github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc42048c1c0, 0xc420c4a3c0, 0xc420c4a3c0, 0x5a0a5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:326 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x6a61900, 0xc42048c1c0, 0xc42001d000, 0xc4206a42d0, 0x6a552a0, 0xc420c4a3c0, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x35 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x6a552a0, 0xc420c4a3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x66 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42048c220, 0xc421163e60, 0xc420c16600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 947 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc420bd14f0, 0xc4203e4250, 0x5, 0x5, 0x1, 0xc425529ae8, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc420bd14f0, 0xc4203e4250, 0x5, 0x5, 0x3, 0xc420029900, 0xc420029970) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc420db8330, 0xc4203e4250, 0x5, 0x5, 0x407883e, 0xc420b01278, 0xc424732a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc420db8330, 0xc4203e4250, 0x5, 0x5, 0x5, 0xc4206c0780, 0xc420b01180, 0x5) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc420db8330, 0xc4203e4250, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc420b01180, 0xc4203e4250, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc4203e4240, 0x7fffffff, 0x0, 0x0, 0xc425529d90, 0x40147f7, 0xc421fb65a0, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc4203e4240, 0x6a54aa0, 0x6cf0d10, 0xc420b01180, 0x0, 0x0, 0x5b333c0, 0xc4224335a0, 0x7fffffff, 0xc421fb65a0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420888be0, 0x5b333c0, 0xc4224335a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:698 +0x12e github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc4212c8810, 0x6a552a0, 0xc420db83c0, 0xc422af3ba0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:394 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x6a61960, 0xc4212c8810, 0xc420518000, 0x6a552a0, 0xc420db83c0, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x83 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x6a552a0, 0xc420db83c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x5d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4212c8830, 0xc4204ba360, 0xc420db8390) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1270 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func4(0x6a61b40, 0xc421624780, 0xc420b18780, 0xc42245c480, 0xc4224c9cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:284 +0x1cd created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:283 +0xaaa goroutine 1208 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc422447900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc422447900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 2012 [IO wait]: internal/poll.runtime_pollWait(0x938e898, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4231b0418, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4231b0418, 0xc422b79000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4231b0400, 0xc422b79000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4231b0400, 0xc422b79000, 0x400, 0x400, 0x301, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b04e8, 0xc422b79000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc422ca8be0, 0xc422b79000, 0x400, 0x400, 0x5b473a0, 0xc4226c8b00, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4226c8bd0, 0x73ccbf8, 0xc422ca8be0, 0x5, 0xc422ca8be0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f3ca80, 0x5d2d117, 0xc421f3cba0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f3ca80, 0xc420b9a000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420cf7f80, 0xc4226fcc78, 0x9, 0x9, 0x9, 0x0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420cf7f80, 0xc4226fcc78, 0x9, 0x9, 0x9, 0xc422ba7d60, 0x400436e, 0xc422ba7e0f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420cf7f80, 0xc4226fcc78, 0x9, 0x9, 0x44f790b, 0xc422745e0c, 0xc423d97df0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4226fcc78, 0x9, 0x9, 0x6a34360, 0xc420cf7f80, 0x0, 0x0, 0xc422745e00, 0xc422ba7e90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4226fcc40, 0xc422745e00, 0xc422745e00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc422eea780, 0xc4228d2720, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4205e8420, 0x6a60d60, 0xc422eea780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc422eb3120, 0xc4205e8420, 0x73ccf20, 0xc422eea780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 1252 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc4219bc480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1323 +0x14b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 114620 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422facb40, 0x45563f3, 0xc422c731e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422facb40, 0x5d2c558, 0xc422c73420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4233d8c30, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4233d8c30, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4248826f8, 0x5b6ec40, 0xc4219cfe60, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423e251c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4248826f8, 0x5b6ec40, 0xc4219cfe60, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc423e251c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4248826f8, 0x5b6ec40, 0xc4219cfe60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423e251c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4248826f8, 0x5b6ec40, 0xc4219cfe60, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80c50, 0x6a551e0, 0xc423e251c0, 0xc4248826f8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x8003f5d2e) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422616d90, 0xc4248826e0, 0x6a551e0, 0xc423e251c0, 0xc400000008, 0xc422cef280, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422616d90, 0x6a551e0, 0xc423e251c0, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422616d90, 0x6a551e0, 0xc423e251c0, 0xc4248822c0, 0xc4219cfda0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 1203 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc420518f00, 0xc400000002, 0xc42168c120, 0xc420c41a40, 0x6a61900, 0xc4203f2ea0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:517 +0x54a github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc420518f00, 0x2, 0x6a552a0, 0xc42130ff80, 0x0, 0x0, 0xc422313ae0, 0xc42168c120, 0xc420c41a40, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x2e2 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc420518f00, 0x6a552a0, 0xc42130ff80, 0x2, 0xc42168c120, 0xc420c41a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:463 +0xd0 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func1.1(0x6a552a0, 0xc42130ff80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:591 +0x60 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9d40, 0xc420f747e0, 0xc42130ff50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 2077 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc423b448d0, 0xc420518e60, 0x7, 0xc423080c80, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc423b448d0, 0x7, 0xc423080c80, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc423b448d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9de00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc423b448d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9de00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc423b448d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9de00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc423b448d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9dd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42293a540, 0x6a552a0, 0xc423b44780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9dd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc422e31100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9dd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc421fea1a0, 0x6a551e0, 0xc422e31100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc422e31100, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc422e31100, 0xc4230a9878, 0xc424a72a00, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc421fea1a0, 0x6a551e0, 0xc422e31100, 0xc424a72a00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc42204b3d0, 0x6a551e0, 0xc422e31100, 0xc421ef75e0, 0x13, 0x20, 0xcbd, 0x6a651a0, 0xc4211df800, 0xc4231bb6c0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4228bb4d0, 0x6a551e0, 0xc422e31100, 0xc42204b440, 0xc42204b3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4228bb4d0, 0xc421be6aa8, 0xc4230aab28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc422cef680, 0x5c46757, 0xc, 0x6a4c120, 0xc4228bb4d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc422cef680, 0x6a4c120, 0xc4228bb4d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc422e31100, 0x6a4c120, 0xc4228bb4d0, 0x5d28ea0, 0x0, 0x0, 0xc4228bb500, 0xc422c0ccc0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc422e31100, 0xc42204b440, 0xc42204b3d0, 0x6a4c120, 0xc4228bb4d0, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc422e31100, 0xc42204b440, 0xc42204b3d0, 0x6a4c120, 0xc4228bb4d0, 0x0, 0xc422c0ccc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc42204b648, 0x6a551e0, 0xc422e31100, 0xc42204b440, 0xc42204b3d0, 0xf25427f44d92d308, 0xc4230ab590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc42204b000, 0x6a551e0, 0xc422e31100, 0xc42204b3d0, 0x1, 0x73e91b0, 0xc421c42bd0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc42204b000, 0x6a551e0, 0xc422e31100, 0x6a568e0, 0xc42353bc80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc42204b000, 0x6a551e0, 0xc422e31100, 0x6a568e0, 0xc42353bc80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc42204b000, 0x6a551e0, 0xc422e31100, 0x6a568e0, 0xc42353bc80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc42204b000, 0x6a551e0, 0xc42132bc00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc42132bc00, 0x0, 0x0, 0xc421870022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423201510, 0xc42100a000, 0x6a551e0, 0xc42132bc00, 0xc423226540, 0x5400, 0x15000, 0xc42136a190, 0xc422d486c0, 0xc423201500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1253 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc4206a42d0, 0x6a61960, 0xc420426e30, 0x6a42020, 0xc4206a42d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:359 +0x215 github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x5c01800, 0xc4206a42d0, 0x6a5eea0, 0xc4213cf5e0, 0x4014f58, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:375 +0xb2 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5c01800, 0xc4206a42d0, 0x6a5eea0, 0xc4213cf5e0, 0xc420b89780, 0x5d2b180, 0x5d2c4d0, 0xc4213cf0e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x103 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc421e177c0, 0xc420ea81e0, 0x6c88b00, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1060 +0x3d6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc421e177c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1147 +0x14c1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc421e177c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 1246 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420cf1cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 114796 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42275bcb0, 0xc4206a4230, 0x7, 0xc420dad9a0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42275bcb0, 0x7, 0xc420dad9a0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42275bcb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42275bcb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42275bcb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42275bcb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4237cea80, 0x6a552a0, 0xc42275bc80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42486fd00, 0xc42194aa00, 0x0, 0xc42486fe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42194aa00, 0x1, 0xc42275bc20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421e1b440, 0xc421bc0000, 0x5, 0x5, 0x15179d5718c80508, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42226c4e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 2011 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc422eea780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 114466 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbfb30, 0x15179d5711ba66c0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbfb30, 0x15179d5711ba66c0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423fbfb30, 0x15179d5711ba66c0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423fbfb00, 0x15179d5711ba66c0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423fbf9b0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423fbf9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423fbf9b0, 0x5c4d11c, 0x10, 0xc421ffd9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423fbf9b0, 0xc421b52a68, 0xc423fbf9b0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423fbf9b0, 0xc421b52a68, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212e6cb0, 0xc421b52a50, 0x6a551e0, 0xc42335b300, 0xc400000008, 0xc421451300, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212e6cb0, 0x6a551e0, 0xc42335b300, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212e6cb0, 0x6a551e0, 0xc42335b300, 0xc421b52840, 0xc421ed4960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 1245 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4224c9680, 0x1, 0xc4219a3380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc4224c9680, 0xc4219a3380, 0x30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:271 +0x2b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc4226f99e0, 0x5b334a0, 0xc4219f60f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:428 +0x81a github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc4203f2ea0, 0xc4219f60c0, 0xc4219f60c0, 0x5a0a5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:326 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x6a61900, 0xc4203f2ea0, 0xc420c41a40, 0xc420518f00, 0x6a552a0, 0xc4219f60c0, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x35 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x6a552a0, 0xc4219f60c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x66 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4203f2f50, 0xc420f747e0, 0xc4219f6060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1254 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc42163a7d0, 0xc420b89770, 0x5, 0x5, 0x0, 0xc421923ae8, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc42163a7d0, 0xc420b89770, 0x5, 0x5, 0x3, 0xc420027300, 0xc420027370) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc4219b8f30, 0xc420b89770, 0x5, 0x5, 0x407883e, 0xc421e178b8, 0xc422ef5280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc4219b8f30, 0xc420b89770, 0x5, 0x5, 0x5, 0xc4219bc300, 0xc421e177c0, 0x5) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc4219b8f30, 0xc420b89770, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc421e177c0, 0xc420b89770, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc420b89760, 0x7fffffff, 0x0, 0x0, 0xc421923d90, 0x40147f7, 0xc4243d6420, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc420b89760, 0x6a54aa0, 0x6cf0d10, 0xc421e177c0, 0x0, 0x0, 0x5b333c0, 0xc42297b860, 0x7fffffff, 0xc4243d6420, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc4213cf5e0, 0x5b333c0, 0xc42297b860, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:698 +0x12e github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc420426e30, 0x6a552a0, 0xc4219b8fc0, 0xc422040b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:394 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x6a61960, 0xc420426e30, 0xc4206a42d0, 0x6a552a0, 0xc4219b8fc0, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x83 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x6a552a0, 0xc4219b8fc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x5d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426e60, 0xc421163e60, 0xc4219b8f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1954 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc422b15fb0, 0xc4202d8b90, 0x7, 0xc4215c33b0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc422b15fb0, 0x7, 0xc4215c33b0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc422b15fb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7cb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc422b15fb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7cb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc422b15fb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7cb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc422b15fb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ee4a80, 0x6a552a0, 0xc422b15f20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc421452c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4230b0d00, 0x6a551e0, 0xc421452c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc421452c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc421452c40, 0xc424033878, 0xc422c5c700, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc4230b0d00, 0x6a551e0, 0xc421452c40, 0xc422c5c700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422fa63d0, 0x6a551e0, 0xc421452c40, 0xc4218c07e0, 0x13, 0x20, 0xccf, 0x6a651a0, 0xc423944300, 0xc423156550, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc423b45830, 0x6a551e0, 0xc421452c40, 0xc422fa6440, 0xc422fa63d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc423b45830, 0xc4215b53a8, 0xc424034b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc4220a92c0, 0x5c46757, 0xc, 0x6a4c120, 0xc423b45830) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc4220a92c0, 0x6a4c120, 0xc423b45830) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc421452c40, 0x6a4c120, 0xc423b45830, 0x5d28ea0, 0x0, 0x0, 0xc423b45860, 0xc4213794e0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc421452c40, 0xc422fa6440, 0xc422fa63d0, 0x6a4c120, 0xc423b45830, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc421452c40, 0xc422fa6440, 0xc422fa63d0, 0x6a4c120, 0xc423b45830, 0x0, 0xc4213794e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422fa6648, 0x6a551e0, 0xc421452c40, 0xc422fa6440, 0xc422fa63d0, 0xc746e6eed560685, 0xc424035590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422fa6000, 0x6a551e0, 0xc421452c40, 0xc422fa63d0, 0x1, 0x73e91b0, 0xc42286e510, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422fa6000, 0x6a551e0, 0xc421452c40, 0x6a568e0, 0xc4235b8d20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422fa6000, 0x6a551e0, 0xc421452c40, 0x6a568e0, 0xc4235b8d20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422fa6000, 0x6a551e0, 0xc421452c40, 0x6a568e0, 0xc4235b8d20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422fa6000, 0x6a551e0, 0xc421155440, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc421155440, 0x0, 0x0, 0xc421e9504a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422e80c40, 0xc4204a4700, 0x6a551e0, 0xc421155440, 0xc4224d4000, 0x5400, 0x15000, 0xc420724a90, 0xc422f7a010, 0xc422e80c30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1963 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc4223d4240, 0xc420518e60, 0x7, 0xc4210fc5f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc4223d4240, 0x7, 0xc4210fc5f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc4223d4240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc4223d4240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc4223d4240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc4223d4240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6cb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421f4a1c0, 0x6a552a0, 0xc4223d41e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6cb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc423aa0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6cb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423183a00, 0x6a551e0, 0xc423aa0d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc423183a00, 0x6a551e0, 0xc423aa0d80, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc423183a00, 0x6a551e0, 0xc423aa0d80, 0xc423183a00, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc423183a00, 0x6a551e0, 0xc423aa0d80, 0xc4232c9bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422ffb000, 0x6a551e0, 0xc423aa0d80, 0x6a568e0, 0xc42390a1e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422ffb000, 0x6a551e0, 0xc423aa0d80, 0x6a568e0, 0xc42390a1e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422ffb000, 0x6a551e0, 0xc421155d80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc421155d80, 0x0, 0x0, 0xc422b7f00d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422e80f70, 0xc42100a000, 0x6a551e0, 0xc421155d80, 0xc4224d4a80, 0x5400, 0x15000, 0xc42136a190, 0xc422f7a370, 0xc422e80f60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1267 [IO wait]: internal/poll.runtime_pollWait(0x73cba00, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420463998, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420463998, 0xc42259d000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420463980, 0xc42259d000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420463980, 0xc42259d000, 0x400, 0x400, 0x0, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0e50, 0xc42259d000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4219f7560, 0x77aa1a0, 0xc4205b0e50, 0x5, 0xc4205b0e50, 0xc421924a60) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224f6a80, 0x5d2d117, 0xc4224f6ba0, 0x40147f7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224f6a80, 0xc422470000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422324840, 0xc42246e118, 0x9, 0x9, 0x0, 0x402e408, 0xc421945880) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422324840, 0xc42246e118, 0x9, 0x9, 0x9, 0xc4231ba020, 0xc421924e20, 0x454a13e) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422324840, 0xc42246e118, 0x9, 0x9, 0x5d2c5d8, 0x200050100, 0xc400000059) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42246e118, 0x9, 0x9, 0x6a34360, 0xc422324840, 0x0, 0x0, 0x0, 0xc423f803b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42246e0e0, 0xc4238c6d80, 0xc4238c6d80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc421624780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 114934 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42387bf80, 0xc4206a4230, 0x7, 0xc422673400, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42387bf80, 0x7, 0xc422673400, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42387bf80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42387bf80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42387bf80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42387bf80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ee4c40, 0x6a552a0, 0xc42387bf50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a55260, 0xc4200cd9e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a55260, 0xc4200cd9e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a55260, 0xc4200cd9e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a55260, 0xc4200cd9e0, 0xc424877c98, 0xc4227ec000, 0x0, 0xc424877d58) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a55260, 0xc4200cd9e0, 0xc4227ec000, 0x33c0, 0x252ad) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*raftLogQueue).process(0xc4204108d0, 0x6a55260, 0xc4200cd9e0, 0xc4224e6700, 0x0, 0x0, 0x0, 0x6a652e0, 0xc420b88320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_log_queue.go:266 +0x266 github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processReplica(0xc420f2d300, 0x6a552a0, 0xc42387b770, 0xc4224e6700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:723 +0x384 github.com/cockroachdb/cockroach/pkg/storage.(*baseQueue).processLoop.func1.2(0x6a552a0, 0xc42387b770) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/queue.go:606 +0xb8 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc42387b770, 0xc422f3dfb0, 0x2b, 0x0, 0x0, 0xc42297b9c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 1761 [IO wait]: internal/poll.runtime_pollWait(0x938ec58, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208ba498, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208ba498, 0xc42259dc00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208ba480, 0xc42259dc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208ba480, 0xc42259dc00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f83e0, 0xc42259dc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420f445a0, 0xc42259dc00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420f2f470, 0x73ccbf8, 0xc420f445a0, 0x5, 0xc420f445a0, 0xc0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4226f6e00, 0x5d2d117, 0xc4226f6f20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4226f6e00, 0xc4230b8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422b19620, 0xc4230b8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc421a80ca0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc421a80ca0, 0xbe9de8a32a893c88, 0x7cae289e7, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc421a80dd0, 0x6a46020, 0xc421a80ca0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc421a80c40, 0x6a551e0, 0xc422f9e740, 0xc4230062d0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4208a4700, 0x6a611e0, 0xc4226f6e00, 0x0, 0x0, 0xc42272604a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4208a4700, 0x6a611e0, 0xc4226f6e00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420f2f3e0, 0x6a61a80, 0xc420f445a0, 0xc420351d40, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc420f445a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc420f445a0, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1268 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc420cf1dc0, 0xc4219fc3f0, 0xc422805fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc421624780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 1255 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue(0xc4206a42d0, 0xc400000003, 0xc4215c9380, 0xc421098840, 0x6a61900, 0xc42043ef30, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:517 +0x54a github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess.func1(0xc4206a42d0, 0x3, 0x6a552a0, 0xc4219b9350, 0x0, 0x0, 0xc4213cf680, 0xc4215c9380, 0xc421098840, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:462 +0x2e2 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).connectAndProcess(0xc4206a42d0, 0x6a552a0, 0xc4219b9350, 0x3, 0xc4215c9380, 0xc421098840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:463 +0xd0 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).SendAsync.func1.1(0x6a552a0, 0xc4219b9350) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:591 +0x60 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420426f40, 0xc421163e60, 0xc4219b9320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1913 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc423009890, 0xc420518e60, 0x7, 0xc42411fae0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc423009890, 0x7, 0xc42411fae0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc423009890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420f2dd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc423009890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420f2dd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc423009890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420f2dd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc423009890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2b700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e6bdc0, 0x6a552a0, 0xc424596d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2b700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc422ef43c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2b700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc424b204e0, 0x6a551e0, 0xc422ef43c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc424b204e0, 0x6a551e0, 0xc422ef43c0, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc424b204e0, 0x6a551e0, 0xc422ef43c0, 0xc424b204e0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc424b204e0, 0x6a551e0, 0xc422ef43c0, 0xc4249e1bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422e82000, 0x6a551e0, 0xc422ef43c0, 0x6a568e0, 0xc42414f800, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422e82000, 0x6a551e0, 0xc422ef43c0, 0x6a568e0, 0xc42414f800, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422e82000, 0x6a551e0, 0xc4211542c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc4211542c0, 0x0, 0x0, 0xc421fc504a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422877ff0, 0xc42100a000, 0x6a551e0, 0xc4211542c0, 0xc42054bc00, 0x5400, 0x15000, 0xc42136a190, 0xc422328440, 0xc422877fe0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1271 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4224c9cc0, 0xc42064d601, 0xc421a1c6c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).RecvCompress(0xc4224c9cc0, 0xc421a1c6c0, 0x30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:271 +0x2b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*clientStream).RecvMsg(0xc42245c480, 0x5b334a0, 0xc422027260, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:428 +0x81a github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchClient).Recv(0xc42043ef30, 0xc422027230, 0xc422027230, 0x5a0a5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:326 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1.1(0x6a61900, 0xc42043ef30, 0xc421098840, 0xc4206a42d0, 0x6a552a0, 0xc422027230, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:491 +0x35 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).processQueue.func1.1(0x6a552a0, 0xc422027230) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:506 +0x66 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc42043efc0, 0xc421163e60, 0xc4219fc870) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1269 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc421624780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1323 +0x14b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 1325 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc4222c9ce0, 0xc4222c9c80, 0xc422021fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc421625c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 1746 [IO wait]: internal/poll.runtime_pollWait(0x73e4550, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420463918, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420463918, 0xc420f0a400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420463900, 0xc420f0a400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420463900, 0xc420f0a400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef190, 0xc420f0a400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42266f2c0, 0xc420f0a400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421012510, 0x73ccbf8, 0xc42266f2c0, 0x5, 0xc42266f2c0, 0xc5) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224e6380, 0x5d2d117, 0xc4224e64a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224e6380, 0xc422e7e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e25b00, 0xc422e7e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054bc60) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054bc60, 0xbe9de8a30869e288, 0x7a8c37466, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054bd90, 0x6a46020, 0xc42054bc60, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054bc00, 0x6a551e0, 0xc421154280, 0xc422877fc0, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420061d40, 0x6a611e0, 0xc4224e6380, 0x0, 0x0, 0xc421fc504a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420061d40, 0x6a611e0, 0xc4224e6380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4210e21b0, 0x6a61a80, 0xc42266f2c0, 0xc421fd8750, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42266f2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42266f2c0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1656 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eb768, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420462998, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420462998, 0xc42106c000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420462980, 0xc42106c000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420462980, 0xc42106c000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e380, 0xc42106c000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc421013f80, 0x77aa1a0, 0xc42000e380, 0x5, 0xc42000e380, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206a6700, 0x5d2d117, 0xc4206a6820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206a6700, 0xc42106a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ac6c0, 0xc4223102e0, 0x5, 0x200, 0xc421bb96e8, 0x6, 0xc4206a68d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ac6c0, 0xc4223102e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422c2cce0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ac6c0, 0xc4223102e0, 0x5, 0x200, 0x77b4618, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc4223102c0, 0xc422c2cce0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc4223102c0, 0xc422c2cce0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc4223102c0, 0xc421bb9890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc4223102c0, 0xc4226c69b0, 0x4b, 0xc421bb9960, 0xc421bb9910, 0x4047467, 0x0, 0x5a02440, 0xc421bb9901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc4223102c0, 0xc4226c69b0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc4223102c0, 0x6a55220, 0xc4200180f8, 0xc4226c69b0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc4223102c0, 0xc4226c69b0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420f716c0, 0xc421bb9c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420f716c0, 0xc421bb9d48, 0xc4226c69b0, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4226c69b0, 0x4b, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4226c69b0, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4226c69b0, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc422c2ccc0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x2, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1786 [IO wait]: internal/poll.runtime_pollWait(0x73ea0a8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb598, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb598, 0xc4220cd400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb580, 0xc4220cd400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb580, 0xc4220cd400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f4b0, 0xc4220cd400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f7f180, 0xc4220cd400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421012420, 0x73ccbf8, 0xc421f7f180, 0x5, 0xc421f7f180, 0xa1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42197e700, 0x5d2d117, 0xc42197e820, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42197e700, 0xc422ee6000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e27a80, 0xc422ee6000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42117d720) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42117d720, 0xbe9de8a325893508, 0x7c5e28cfb, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42117d850, 0x6a46020, 0xc42117d720, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42117d6c0, 0x6a551e0, 0xc42104bb80, 0xc4215edb00, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4203b57c0, 0x6a611e0, 0xc42197e700, 0x0, 0x0, 0xc421c2e04a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc4203b57c0, 0x6a611e0, 0xc42197e700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222be540, 0x6a61a80, 0xc421f7f180, 0xc4203a3a90, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f7f180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc421f7f180, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1349 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc421fc2780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc421fc2780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 1257 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc420420a00, 0xc42163a960, 0xc42035ffb0, 0xf, 0xc4226d66c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:581 +0x42c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x6a552a0, 0xc4219b94a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:512 +0x74 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc4204278b0, 0xc421163e60, 0xc4219b9470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1348 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc421154b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 1244 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream.func4(0x6a61b40, 0xc4219bc480, 0xc422447680, 0xc4226f99e0, 0xc4224c9680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:284 +0x1cd created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newClientStream /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:283 +0xaaa goroutine 1256 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc4219b9440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 1247 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc422234c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc422234c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 1223 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).read(0xc4215c3810, 0xc421209b50, 0x5, 0x5, 0x1, 0xc423387ae8, 0x402e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:132 +0x28b github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*recvBufferReader).Read(0xc4215c3810, 0xc421209b50, 0x5, 0x5, 0x3, 0xc420028600, 0xc420028670) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:121 +0x67 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*transportReader).Read(0xc421a3a3c0, 0xc421209b50, 0x5, 0x5, 0x407883e, 0xc4216c0378, 0xc4226c5200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:395 +0x55 io.ReadAtLeast(0x6a3aee0, 0xc421a3a3c0, 0xc421209b50, 0x5, 0x5, 0x5, 0xc421624600, 0xc4216c0280, 0x5) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a3aee0, 0xc421a3a3c0, 0xc421209b50, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Read(0xc4216c0280, 0xc421209b50, 0x5, 0x5, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:379 +0xbf github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*parser).recvMsg(0xc421209b40, 0x7fffffff, 0x0, 0x0, 0xc423387d90, 0x40147f7, 0xc424200b40, 0x60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:285 +0x65 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recv(0xc421209b40, 0x6a54aa0, 0x6cf0d10, 0xc4216c0280, 0x0, 0x0, 0x5b333c0, 0xc4223a0d20, 0x7fffffff, 0xc424200b40, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/rpc_util.go:394 +0x4d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*serverStream).RecvMsg(0xc420e09180, 0x5b333c0, 0xc4223a0d20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/stream.go:698 +0x12e github.com/cockroachdb/cockroach/pkg/storage.(*multiRaftRaftMessageBatchServer).Recv(0xc420362d40, 0x6a552a0, 0xc421a3a450, 0xc42282bd40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:394 +0x62 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1.1(0x6a61960, 0xc420362d40, 0xc420518f00, 0x6a552a0, 0xc421a3a450, 0x18, 0x481b690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:330 +0x83 github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch.func1.1(0x6a552a0, 0xc421a3a450) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:353 +0x5d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420362d60, 0xc420f747e0, 0xc421a3a420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1222 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*RaftTransport).RaftMessageBatch(0xc420518f00, 0x6a61960, 0xc420362d40, 0x6a42020, 0xc420518f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft_transport.go:359 +0x215 github.com/cockroachdb/cockroach/pkg/storage._MultiRaft_RaftMessageBatch_Handler(0x5c01800, 0xc420518f00, 0x6a5eea0, 0xc420e09180, 0x4014f58, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/raft.pb.go:375 +0xb2 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func2(0x5c01800, 0xc420518f00, 0x6a5eea0, 0xc420e09180, 0xc421209b60, 0x5d2b180, 0xc420bc8ecc, 0x4a24c84) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:182 +0x103 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processStreamingRPC(0xc4205e91e0, 0x6a60d60, 0xc421624600, 0xc4216c0280, 0xc420e89500, 0x6c88b00, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1060 +0x3d6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e91e0, 0x6a60d60, 0xc421624600, 0xc4216c0280, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1147 +0x14c1 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421edaa70, 0xc4205e91e0, 0x6a60d60, 0xc421624600, 0xc4216c0280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 1205 [select]: github.com/cockroachdb/cockroach/pkg/rpc.(*Context).runHeartbeat(0xc42115a780, 0xc420ebf590, 0xc42035f7d0, 0xf, 0xc420bb7c80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:581 +0x42c github.com/cockroachdb/cockroach/pkg/rpc.(*Context).GRPCDial.func1.1.1(0x6a552a0, 0xc42130c0f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:512 +0x74 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker.func1(0xc420bf9f10, 0xc420f747e0, 0xc42130c0c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:195 +0xf3 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunWorker /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:188 +0xad goroutine 1207 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc420739d40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 1250 [IO wait]: internal/poll.runtime_pollWait(0x73cbb80, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420713198, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420713198, 0xc423d4f800, 0x800, 0x800) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420713180, 0xc423d4f800, 0x800, 0x800, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420713180, 0xc423d4f800, 0x800, 0x800, 0x6a, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000eb58, 0xc423d4f800, 0x800, 0x800, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4212cfcb0, 0x77aa1a0, 0xc42000eb58, 0x5, 0xc42000eb58, 0xc421927a60) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42251c000, 0x5d2d117, 0xc42251c120, 0x40147f7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42251c000, 0xc421a4c000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4215c9020, 0xc421e931b8, 0x9, 0x9, 0x0, 0x402e408, 0xc420034700) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4215c9020, 0xc421e931b8, 0x9, 0x9, 0x9, 0xc423f643e0, 0xc421927e20, 0x454a13e) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4215c9020, 0xc421e931b8, 0x9, 0x9, 0x5d2c5d8, 0x200050100, 0xc4000060cd) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc421e931b8, 0x9, 0x9, 0x6a34360, 0xc4215c9020, 0x0, 0x0, 0x0, 0xc421ac8528) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc421e93180, 0xc423919a10, 0xc423919a10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc4219bc480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 1955 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc42248e690, 0xc420518e60, 0x7, 0xc420b7b4f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc42248e690, 0x7, 0xc420b7b4f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc42248e690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42374db00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc42248e690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42374db00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc42248e690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42374db00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc42248e690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42374da00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423073340, 0x6a552a0, 0xc42248e630, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42374da00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc4241cc200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42374da00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc420c3f520, 0x6a551e0, 0xc4241cc200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4241cc200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4241cc200, 0xc4227fb878, 0xc422a09100, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc420c3f520, 0x6a551e0, 0xc4241cc200, 0xc422a09100, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422fa73d0, 0x6a551e0, 0xc4241cc200, 0xc422682180, 0x13, 0x20, 0xcc2, 0x6a651a0, 0xc421bc0600, 0xc423678a30, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc422b15a40, 0x6a551e0, 0xc4241cc200, 0xc422fa7440, 0xc422fa73d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc422b15a40, 0xc4223ec1a8, 0xc4227fcb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42456e580, 0x5c46757, 0xc, 0x6a4c120, 0xc422b15a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42456e580, 0x6a4c120, 0xc422b15a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4241cc200, 0x6a4c120, 0xc422b15a40, 0x5d28ea0, 0x0, 0x0, 0xc422b15a70, 0xc42037d6a0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4241cc200, 0xc422fa7440, 0xc422fa73d0, 0x6a4c120, 0xc422b15a40, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4241cc200, 0xc422fa7440, 0xc422fa73d0, 0x6a4c120, 0xc422b15a40, 0x0, 0xc42037d6a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422fa7648, 0x6a551e0, 0xc4241cc200, 0xc422fa7440, 0xc422fa73d0, 0x502e84dc51603e03, 0xc4227fd590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422fa7000, 0x6a551e0, 0xc4241cc200, 0xc422fa73d0, 0x1, 0x73e91b0, 0xc422e01e60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422fa7000, 0x6a551e0, 0xc4241cc200, 0x6a568e0, 0xc420de7980, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422fa7000, 0x6a551e0, 0xc4241cc200, 0x6a568e0, 0xc420de7980, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422fa7000, 0x6a551e0, 0xc4241cc200, 0x6a568e0, 0xc420de7980, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422fa7000, 0x6a551e0, 0xc421155500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc421155500, 0x0, 0x0, 0xc421c2f022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422e80ca0, 0xc42100a000, 0x6a551e0, 0xc421155500, 0xc4224d41c0, 0x5400, 0x15000, 0xc42136a190, 0xc422f7a020, 0xc422e80c90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1211 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a552a0, 0xc4219b8a50, 0xc4219b89f0, 0xc421922fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server.func2(0xc4219bc300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:261 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:260 +0xee9 goroutine 1212 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc4219bc300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 1213 [IO wait]: internal/poll.runtime_pollWait(0x73cbac0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420713218, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420713218, 0xc421fe4000, 0x2000, 0x2000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420713200, 0xc421fe4000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420713200, 0xc421fe4000, 0x2000, 0x2000, 0x1f01, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0e20, 0xc421fe4000, 0x2000, 0x2000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc422313b80, 0xc421fe4000, 0x2000, 0x2000, 0x5b473a0, 0xc4219b8400, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4219b84b0, 0x73ccbf8, 0xc422313b80, 0x5, 0xc422313b80, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42251c380, 0x5d2d117, 0xc42251c4a0, 0x4007d18) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42251c380, 0xc422542000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4215c8f00, 0xc421e92ff8, 0x9, 0x9, 0xc423e25480, 0x1000f01223ec600, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4215c8f00, 0xc421e92ff8, 0x9, 0x9, 0x9, 0x45472e6, 0xc4219b8e70, 0xc422e9d020) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4215c8f00, 0xc421e92ff8, 0x9, 0x9, 0xc422e9d020, 0x59, 0x59) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc421e92ff8, 0x9, 0x9, 0x6a34360, 0xc4215c8f00, 0x0, 0x100000000, 0x300000059, 0x5900000001) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc421e92fc0, 0xc4219b8900, 0xc4219b8900, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc4219bc300, 0xc42130c8d0, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4205e8420, 0x6a60d60, 0xc4219bc300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc4203718e0, 0xc4205e8420, 0x73ccf20, 0xc4219bc300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 1282 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc421624600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 1283 [IO wait]: internal/poll.runtime_pollWait(0x73cb940, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420713298, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420713298, 0xc421e9e000, 0x8000, 0x8000) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420713280, 0xc421e9e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420713280, 0xc421e9e000, 0x8000, 0x8000, 0x1, 0xc422801898, 0x404b58a) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0e48, 0xc421e9e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc422313c20, 0xc421e9e000, 0x8000, 0x8000, 0x5b473a0, 0xc42130cb00, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc42130cbd0, 0x73ccbf8, 0xc422313c20, 0x5, 0xc422313c20, 0xf) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224f6e00, 0x5d2d117, 0xc4224f6f20, 0x30) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224f6e00, 0xc42245e000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422324720, 0xc4226fded8, 0x9, 0x9, 0x1, 0xc423919680, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422324720, 0xc4226fded8, 0x9, 0x9, 0x9, 0x45472e6, 0xc423919590, 0x0) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422324720, 0xc4226fded8, 0x9, 0x9, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4226fded8, 0x9, 0x9, 0x6a34360, 0xc422324720, 0x0, 0x1000100000000, 0x5900000060, 0x6000010001) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4226fdea0, 0xc4219fc0c0, 0xc4219fc0c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421624600, 0xc42130d5f0, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4205e91e0, 0x6a60d60, 0xc421624600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc420371e20, 0xc4205e91e0, 0x73ccf20, 0xc421624600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 1335 [IO wait]: internal/poll.runtime_pollWait(0x73cb880, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c33118, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c33118, 0xc4222b8c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c33100, 0xc4222b8c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c33100, 0xc4222b8c00, 0x400, 0x400, 0x2e, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206eee58, 0xc4222b8c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4222bed20, 0x77aa1a0, 0xc4206eee58, 0x5, 0xc4206eee58, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c59880, 0x5d2d117, 0xc421c599a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c59880, 0xc422888000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc421fc6ea0, 0xc42246f8b8, 0x9, 0x9, 0x9, 0x0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc421fc6ea0, 0xc42246f8b8, 0x9, 0x9, 0x9, 0xc422485de0, 0x400436e, 0xc422485e8f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc421fc6ea0, 0xc42246f8b8, 0x9, 0x9, 0x44f790b, 0xc424b9392c, 0xc4222c2d70) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42246f8b8, 0x9, 0x9, 0x6a34360, 0xc421fc6ea0, 0x0, 0x0, 0xc424b93920, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42246f880, 0xc424b93920, 0xc424b93920, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc421625e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 1725 [IO wait]: internal/poll.runtime_pollWait(0x73ea468, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420463c98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420463c98, 0xc422b79800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420463c80, 0xc422b79800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420463c80, 0xc422b79800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f4b8, 0xc422b79800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f7f360, 0xc422b79800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4222cda40, 0x73ccbf8, 0xc421f7f360, 0x5, 0xc421f7f360, 0xa7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c58700, 0x5d2d117, 0xc421c58820, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c58700, 0xc422af6000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4203716a0, 0xc422af6000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054a060) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054a060, 0xbe9de8a32ce1b730, 0x7cd3b00a7, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054a190, 0x6a46020, 0xc42054a060, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054a000, 0x6a551e0, 0xc421167d00, 0xc4212298e0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4212f1240, 0x6a611e0, 0xc421c58700, 0x0, 0x0, 0xc422d5f035, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4212f1240, 0x6a611e0, 0xc421c58700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222cda10, 0x6a61a80, 0xc421f7f360, 0xc422c11950, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f7f360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc421f7f360, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1914 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc423123740, 0xc420518e60, 0x7, 0xc422b5af50, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc423123740, 0x7, 0xc422b5af50, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc423123740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc424604e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc423123740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc424604e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc423123740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc424604e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc423123740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420ded700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423d30700, 0x6a552a0, 0xc423123680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420ded700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc42141c5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420ded700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc42162b1e0, 0x6a551e0, 0xc42141c5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc42141c5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc42141c5c0, 0xc4231e9878, 0xc4241fea00, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc42162b1e0, 0x6a551e0, 0xc42141c5c0, 0xc4241fea00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422e833d0, 0x6a551e0, 0xc42141c5c0, 0xc422682620, 0x13, 0x20, 0xcc3, 0x6a651a0, 0xc420584000, 0xc424b277b0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4225f86c0, 0x6a551e0, 0xc42141c5c0, 0xc422e83440, 0xc422e833d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4225f86c0, 0xc422452028, 0xc4231eab28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42456f8c0, 0x5c46757, 0xc, 0x6a4c120, 0xc4225f86c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42456f8c0, 0x6a4c120, 0xc4225f86c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc42141c5c0, 0x6a4c120, 0xc4225f86c0, 0x5d28ea0, 0x0, 0x0, 0xc4225f86f0, 0xc422c2d200, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc42141c5c0, 0xc422e83440, 0xc422e833d0, 0x6a4c120, 0xc4225f86c0, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc42141c5c0, 0xc422e83440, 0xc422e833d0, 0x6a4c120, 0xc4225f86c0, 0x0, 0xc422c2d200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422e83648, 0x6a551e0, 0xc42141c5c0, 0xc422e83440, 0xc422e833d0, 0xb1265d357c00c010, 0xc4231eb590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422e83000, 0x6a551e0, 0xc42141c5c0, 0xc422e833d0, 0x1, 0x73e91b0, 0xc422aa7200, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422e83000, 0x6a551e0, 0xc42141c5c0, 0x6a568e0, 0xc423340960, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422e83000, 0x6a551e0, 0xc42141c5c0, 0x6a568e0, 0xc423340960, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422e83000, 0x6a551e0, 0xc42141c5c0, 0x6a568e0, 0xc423340960, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422e83000, 0x6a551e0, 0xc421154380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc421154380, 0x0, 0x0, 0xc4220ee04a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422e80050, 0xc42100a000, 0x6a551e0, 0xc421154380, 0xc42054bdc0, 0x5400, 0x15000, 0xc42136a190, 0xc422328470, 0xc422e80040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1326 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).keepalive(0xc421625c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:956 +0x264 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Server /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:264 +0xf0e goroutine 1336 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc421154c80, 0xc4222c9e60, 0xc422489fb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc421625e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 1726 [IO wait]: internal/poll.runtime_pollWait(0x73e9fe8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb618, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb618, 0xc422b79c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb600, 0xc422b79c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb600, 0xc422b79c00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f4c0, 0xc422b79c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f7f400, 0xc422b79c00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4222cdaa0, 0x73ccbf8, 0xc421f7f400, 0x5, 0xc421f7f400, 0xbf) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c58a80, 0x5d2d117, 0xc421c58ba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c58a80, 0xc420ed3000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4223d73a0, 0xc420ed3000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054b020) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054b020, 0xbe9de8a32c62a3f0, 0x7ccbbf093, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054b150, 0x6a46020, 0xc42054b020, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054afc0, 0x6a551e0, 0xc420cf1b00, 0xc421229fe0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4212f1440, 0x6a611e0, 0xc421c58a80, 0x0, 0x0, 0xc422d7704a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4212f1440, 0x6a611e0, 0xc421c58a80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222cda70, 0x6a61a80, 0xc421f7f400, 0xc422c11980, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f7f400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc421f7f400, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1365 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc4222ccf90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 1337 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc421625e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1323 +0x14b created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 1327 [IO wait]: internal/poll.runtime_pollWait(0x73e4910, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c33198, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c33198, 0xc4222ea800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c33180, 0xc4222ea800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c33180, 0xc4222ea800, 0x400, 0x400, 0x301, 0x0, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f1a8, 0xc4222ea800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f9a500, 0xc4222ea800, 0x400, 0x400, 0x5b473a0, 0xc4222c9700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4222c9740, 0x73ccbf8, 0xc421f9a500, 0x5, 0xc421f9a500, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfaa80, 0x5d2d117, 0xc421bfaba0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfaa80, 0xc422878000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc421fc6d80, 0xc42246f6f8, 0x9, 0x9, 0x9, 0x0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc421fc6d80, 0xc42246f6f8, 0x9, 0x9, 0x9, 0xc422800d60, 0x400436e, 0xc422800e0f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc421fc6d80, 0xc42246f6f8, 0x9, 0x9, 0x44f790b, 0xc42310b54c, 0xc42194df20) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc42246f6f8, 0x9, 0x9, 0x6a34360, 0xc421fc6d80, 0x0, 0x0, 0xc42310b540, 0xc422800e90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc42246f6c0, 0xc42310b540, 0xc42310b540, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Server).HandleStreams(0xc421625c80, 0xc4222bf1d0, 0x5d2c470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_server.go:395 +0x3a github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams(0xc4205e91e0, 0x6a60d60, 0xc421625c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:634 +0xf2 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func2() /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:591 +0x3c github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn.func3(0xc421c03960, 0xc4205e91e0, 0x73ccf20, 0xc421625c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:599 +0x27 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleRawConn /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:598 +0x656 goroutine 1750 [IO wait]: internal/poll.runtime_pollWait(0x73e3c50, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420463a18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420463a18, 0xc421fd1400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420463a00, 0xc421fd1400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420463a00, 0xc421fd1400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef218, 0xc421fd1400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42266f400, 0xc421fd1400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210e2c00, 0x73ccbf8, 0xc42266f400, 0x5, 0xc42266f400, 0xb0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42251ce00, 0x5d2d117, 0xc42251cf20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42251ce00, 0xc421b6d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422c0dac0, 0xc421b6d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc421a80060) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc421a80060, 0xbe9de8a32dd84848, 0x7ce318f14, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc421a80190, 0x6a46020, 0xc421a80060, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc421a80000, 0x6a551e0, 0xc42101dfc0, 0xc422171230, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc421278080, 0x6a611e0, 0xc42251ce00, 0x0, 0x0, 0xc422c60022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc421278080, 0x6a611e0, 0xc42251ce00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4210e2bd0, 0x6a61a80, 0xc42266f400, 0xc421fd9090, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42266f400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc42266f400, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1910 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc423407950, 0xc420518e60, 0x7, 0xc42432f220, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc423407950, 0x7, 0xc42432f220, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc423407950, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc423407950, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc423407950, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc423407950, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdba00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42420a380, 0x6a552a0, 0xc4234078f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdba00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc42456fc40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdba00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc424a44340, 0x6a551e0, 0xc42456fc40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc424a44340, 0x6a551e0, 0xc42456fc40, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc424a44340, 0x6a551e0, 0xc42456fc40, 0xc424a44340, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc424a44340, 0x6a551e0, 0xc42456fc40, 0xc423107bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422dc1000, 0x6a551e0, 0xc42456fc40, 0x6a568e0, 0xc420de7e00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422dc1000, 0x6a551e0, 0xc42456fc40, 0x6a568e0, 0xc420de7e00, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422dc1000, 0x6a551e0, 0xc420fef400, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc420fef400, 0x0, 0x0, 0xc4220ef035, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422876d40, 0xc42100a000, 0x6a551e0, 0xc420fef400, 0xc42054b6c0, 0x5400, 0x15000, 0xc42136a190, 0xc421c40360, 0xc422876d30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114407 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423d659e0, 0xc4206a4230, 0x7, 0xc421056d70, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423d659e0, 0x7, 0xc421056d70, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423d659e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423d659e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423d659e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423d659e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423226a80, 0x6a552a0, 0xc423d659b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42417fd00, 0xc4242a4300, 0x0, 0xc42417fe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4242a4300, 0x1, 0xc423d65950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421b44480, 0xc423945200, 0x5, 0x5, 0x15179d5711a71570, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42275a5d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 1995 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc4225f9890, 0xc420518e60, 0x7, 0xc422be91d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc4225f9890, 0x7, 0xc422be91d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc4225f9890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4234e2200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc4225f9890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4234e2200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc4225f9890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4234e2200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc4225f9890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4234e2100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42293aa80, 0x6a552a0, 0xc4225f97d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4234e2100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc4247337c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4234e2100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422eb9ba0, 0x6a551e0, 0xc4247337c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc422eb9ba0, 0x6a551e0, 0xc4247337c0, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc422eb9ba0, 0x6a551e0, 0xc4247337c0, 0xc422eb9ba0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc422eb9ba0, 0x6a551e0, 0xc4247337c0, 0xc422859bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422ffa000, 0x6a551e0, 0xc4247337c0, 0x6a568e0, 0xc4228acd80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422ffa000, 0x6a551e0, 0xc4247337c0, 0x6a568e0, 0xc4228acd80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422ffa000, 0x6a551e0, 0xc423228e80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc423228e80, 0x0, 0x0, 0xc421871037, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4232003b0, 0xc42100a000, 0x6a551e0, 0xc423228e80, 0xc422fd2a80, 0x5400, 0x15000, 0xc42136a190, 0xc422bdfcd0, 0xc4232003a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1765 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc421064750, 0xc4202d8b90, 0x7, 0xc420bd2c80, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc421064750, 0x7, 0xc420bd2c80, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc421064750, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc421064750, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc421064750, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc421064750, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e07180, 0x6a552a0, 0xc421064690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc42378d640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc42387dba0, 0x6a551e0, 0xc42378d640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc42387dba0, 0x6a551e0, 0xc42378d640, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc42387dba0, 0x6a551e0, 0xc42378d640, 0xc42387dba0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc42387dba0, 0x6a551e0, 0xc42378d640, 0xc422bb5bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422b3d000, 0x6a551e0, 0xc42378d640, 0x6a568e0, 0xc422cd7ce0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422b3d000, 0x6a551e0, 0xc42378d640, 0x6a568e0, 0xc422cd7ce0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422b3d000, 0x6a551e0, 0xc420cf0840, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc420cf0840, 0x0, 0x0, 0xc422c3400d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc421229970, 0xc4204a4700, 0x6a551e0, 0xc420cf0840, 0xc42054a380, 0x5400, 0x15000, 0xc420724a90, 0xc420386ed0, 0xc421229960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1749 [IO wait]: internal/poll.runtime_pollWait(0x73e3a10, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420463c18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420463c18, 0xc421fd1000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420463c00, 0xc421fd1000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420463c00, 0xc421fd1000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef220, 0xc421fd1000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42266f360, 0xc421fd1000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210e2ba0, 0x73ccbf8, 0xc42266f360, 0x5, 0xc42266f360, 0xc9) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42251c700, 0x5d2d117, 0xc42251c820, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42251c700, 0xc4229d8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421f30f60, 0xc4229d8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4203d5aa0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4203d5aa0, 0xbe9de8a321fe6c50, 0x7c257cd22, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4203d5bd0, 0x6a46020, 0xc4203d5aa0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4203d5a40, 0x6a551e0, 0xc420bb9d00, 0xc422939ec0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc421278000, 0x6a611e0, 0xc42251c700, 0x0, 0x0, 0xc422c5a020, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc421278000, 0x6a611e0, 0xc42251c700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4210e2b70, 0x6a61a80, 0xc42266f360, 0xc421fd9060, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42266f360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc42266f360, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1866 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4220274d0, 0xc4206a4230, 0x7, 0xc42350a5f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4220274d0, 0x7, 0xc42350a5f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4220274d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420721d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4220274d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420721d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4220274d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420721d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4220274d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420721c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422fc9500, 0x6a552a0, 0xc422027470, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420721c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc424a5b680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420721c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc42298a680, 0x6a551e0, 0xc424a5b680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc424a5b680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc424a5b680, 0xc423b6b878, 0xc4236b1800, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc42298a680, 0x6a551e0, 0xc424a5b680, 0xc4236b1800, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4229da3d0, 0x6a551e0, 0xc424a5b680, 0xc421ece080, 0x13, 0x20, 0xcb9, 0x6a651a0, 0xc4222bc300, 0xc424345f40, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc420dc7140, 0x6a551e0, 0xc424a5b680, 0xc4229da440, 0xc4229da3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc420dc7140, 0xc4224533a8, 0xc423b6cb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc424a5b940, 0x5c46757, 0xc, 0x6a4c120, 0xc420dc7140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc424a5b940, 0x6a4c120, 0xc420dc7140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc424a5b680, 0x6a4c120, 0xc420dc7140, 0x5d28ea0, 0x0, 0x0, 0xc420dc7170, 0xc422697600, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc424a5b680, 0xc4229da440, 0xc4229da3d0, 0x6a4c120, 0xc420dc7140, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc424a5b680, 0xc4229da440, 0xc4229da3d0, 0x6a4c120, 0xc420dc7140, 0x0, 0xc422697600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc4229da648, 0x6a551e0, 0xc424a5b680, 0xc4229da440, 0xc4229da3d0, 0x9666d2fcb687da7d, 0xc423b6d590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc4229da000, 0x6a551e0, 0xc424a5b680, 0xc4229da3d0, 0x1, 0x73e91b0, 0xc424a6e750, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc4229da000, 0x6a551e0, 0xc424a5b680, 0x6a568e0, 0xc42363bc20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc4229da000, 0x6a551e0, 0xc424a5b680, 0x6a568e0, 0xc42363bc20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4229da000, 0x6a551e0, 0xc424a5b680, 0x6a568e0, 0xc42363bc20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4229da000, 0x6a551e0, 0xc420bb9d40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc420bb9d40, 0x0, 0x0, 0xc422c5a020, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422939ef0, 0xc42100a460, 0x6a551e0, 0xc420bb9d40, 0xc4203d5a40, 0x5400, 0x15000, 0xc4206d5810, 0xc4223e3400, 0xc422939ee0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1717 [IO wait]: internal/poll.runtime_pollWait(0x73e3b90, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb418, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb418, 0xc4220cc400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb400, 0xc4220cc400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb400, 0xc4220cc400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f400, 0xc4220cc400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f7f0e0, 0xc4220cc400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421012300, 0x73ccbf8, 0xc421f7f0e0, 0x5, 0xc421f7f0e0, 0x9c) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f07180, 0x5d2d117, 0xc421f072a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f07180, 0xc422e56000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421fd61c0, 0xc422e56000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc9e20) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc9e20, 0xbe9de8a31b8f4088, 0x7bbe8a9e5, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc9f50, 0x6a46020, 0xc422fc9e20, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc9dc0, 0x6a551e0, 0xc42132ab40, 0xc423201210, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4212f0cc0, 0x6a611e0, 0xc421f07180, 0x0, 0x0, 0xc42248b037, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc4212f0cc0, 0x6a611e0, 0xc421f07180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222cc3f0, 0x6a61a80, 0xc421f7f0e0, 0xc421b23e00, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f7f0e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc421f7f0e0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1773 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42334cae0, 0xc4206a4230, 0x7, 0xc4228e76d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42334cae0, 0x7, 0xc4228e76d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42334cae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6aa00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42334cae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6aa00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42334cae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6aa00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42334cae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6a900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422690fc0, 0x6a552a0, 0xc42334c9f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6a900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc422494140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6a900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc424b21860, 0x6a551e0, 0xc422494140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc422494140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc422494140, 0xc421a8d878, 0xc42235a000, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc424b21860, 0x6a551e0, 0xc422494140, 0xc42235a000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422b453d0, 0x6a551e0, 0xc422494140, 0xc424b93fa0, 0x13, 0x20, 0xcca, 0x6a651a0, 0xc423944600, 0xc423d974e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4210653b0, 0x6a551e0, 0xc422494140, 0xc422b45440, 0xc422b453d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4210653b0, 0xc4219bc028, 0xc421a8eb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc422494640, 0x5c46757, 0xc, 0x6a4c120, 0xc4210653b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc422494640, 0x6a4c120, 0xc4210653b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc422494140, 0x6a4c120, 0xc4210653b0, 0x5d28ea0, 0x0, 0x0, 0xc4210653e0, 0xc422e27a60, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc422494140, 0xc422b45440, 0xc422b453d0, 0x6a4c120, 0xc4210653b0, 0x735a000, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc422494140, 0xc422b45440, 0xc422b453d0, 0x6a4c120, 0xc4210653b0, 0x0, 0xc422e27a60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422b45648, 0x6a551e0, 0xc422494140, 0xc422b45440, 0xc422b453d0, 0x3b87498522a659f8, 0xc421a8f590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422b45000, 0x6a551e0, 0xc422494140, 0xc422b453d0, 0x1, 0x73e91b0, 0xc422faa3f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422b45000, 0x6a551e0, 0xc422494140, 0x6a568e0, 0xc423e0b6e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422b45000, 0x6a551e0, 0xc422494140, 0x6a568e0, 0xc423e0b6e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422b45000, 0x6a551e0, 0xc422494140, 0x6a568e0, 0xc423e0b6e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422b45000, 0x6a551e0, 0xc420cf1b40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc420cf1b40, 0x0, 0x0, 0xc422d7704a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc421e7e010, 0xc42100a460, 0x6a551e0, 0xc420cf1b40, 0xc42054afc0, 0x5400, 0x15000, 0xc4206d5810, 0xc4204130f0, 0xc421e7e000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1764 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423d65920, 0xc4206a4230, 0x7, 0xc420ef4190, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423d65920, 0x7, 0xc420ef4190, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423d65920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412f100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423d65920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412f100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423d65920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412f100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423d65920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412f000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e6a380, 0x6a552a0, 0xc423d658c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412f000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc424a5b800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412f000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc42386e4e0, 0x6a551e0, 0xc424a5b800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc424a5b800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc424a5b800, 0xc42436d878, 0xc424b90a00, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc42386e4e0, 0x6a551e0, 0xc424a5b800, 0xc424b90a00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4212f63d0, 0x6a551e0, 0xc424a5b800, 0xc421ecf2c0, 0x13, 0x20, 0xccb, 0x6a651a0, 0xc421e20000, 0xc42216c230, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4232936e0, 0x6a551e0, 0xc424a5b800, 0xc4212f6440, 0xc4212f63d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4232936e0, 0xc4219bcaa8, 0xc42436eb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc424a5bfc0, 0x5c46757, 0xc, 0x6a4c120, 0xc4232936e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc424a5bfc0, 0x6a4c120, 0xc4232936e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc424a5b800, 0x6a4c120, 0xc4232936e0, 0x5d28ea0, 0x0, 0x0, 0xc423293710, 0xc420eaa980, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc424a5b800, 0xc4212f6440, 0xc4212f63d0, 0x6a4c120, 0xc4232936e0, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc424a5b800, 0xc4212f6440, 0xc4212f63d0, 0x6a4c120, 0xc4232936e0, 0x0, 0xc420eaa980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc4212f6648, 0x6a551e0, 0xc424a5b800, 0xc4212f6440, 0xc4212f63d0, 0xce118c4dd7f7f8ab, 0xc42436f590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc4212f6000, 0x6a551e0, 0xc424a5b800, 0xc4212f63d0, 0x1, 0x73e91b0, 0xc42222f3b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc4212f6000, 0x6a551e0, 0xc424a5b800, 0x6a568e0, 0xc423419020, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc4212f6000, 0x6a551e0, 0xc424a5b800, 0x6a568e0, 0xc423419020, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4212f6000, 0x6a551e0, 0xc424a5b800, 0x6a568e0, 0xc423419020, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4212f6000, 0x6a551e0, 0xc421167e80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc421167e80, 0x0, 0x0, 0xc422d5f035, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc421229910, 0xc42100a460, 0x6a551e0, 0xc421167e80, 0xc42054a000, 0x5400, 0x15000, 0xc4206d5810, 0xc420386e80, 0xc421229900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1723 [IO wait]: internal/poll.runtime_pollWait(0x73e9f28, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb698, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb698, 0xc422b79400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb680, 0xc422b79400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb680, 0xc422b79400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f4c8, 0xc422b79400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f7f220, 0xc422b79400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4222cd500, 0x73ccbf8, 0xc421f7f220, 0x5, 0xc421f7f220, 0xc2) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c58380, 0x5d2d117, 0xc421c584a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c58380, 0xc420ed1000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4223d7300, 0xc420ed1000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054ae60) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054ae60, 0xbe9de8a318bd5bd8, 0x7b916ced6, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054af90, 0x6a46020, 0xc42054ae60, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054ae00, 0x6a551e0, 0xc420cf1a00, 0xc421229f80, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4212f0f80, 0x6a611e0, 0xc421c58380, 0x0, 0x0, 0xc422d3c00d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4212f0f80, 0x6a611e0, 0xc421c58380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222cd4d0, 0x6a61a80, 0xc421f7f220, 0xc422c11080, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f7f220) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc421f7f220, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 115146 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4245626c0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4245626c0, 0x15179d57208582c0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4245626c0, 0x15179d57208582c0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4245626c0, 0x15179d57208582c0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc424562690, 0x15179d57208582c0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4245625d0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4245625d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4245625a0, 0x5c4d11c, 0x10, 0xc423c512c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4245625a0, 0xc422759340, 0xc4245625a0, 0xc42385d568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4245625a0, 0xc422759340, 0xc420b53500, 0xc4241ca108, 0xc4241cc9c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc424562480, 0x5c13340, 0xc422759340, 0x5c13340, 0xc422759340, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc424562480, 0x5c13340, 0xc422759340, 0xc4203b9ae0, 0xc4203b9b60, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc424562480, 0x5c13340, 0xc422759340, 0xc4203b9ae0, 0xc4203b9b60, 0x6, 0xc42385d968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc424562480, 0xc4227591f0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc42487ac40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4239592c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4239592c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4239592c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 1752 [IO wait]: internal/poll.runtime_pollWait(0x7724780, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96118, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96118, 0xc422085400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96100, 0xc422085400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96100, 0xc422085400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e408, 0xc422085400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4221c81e0, 0xc422085400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b23dd0, 0x73ccbf8, 0xc4221c81e0, 0x5, 0xc4221c81e0, 0xc5) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f06a80, 0x5d2d117, 0xc421f06ba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f06a80, 0xc422dbf000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422d3b060, 0xc422dbf000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc9c60) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc9c60, 0xbe9de8a306250368, 0x7a67e9770, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc9d90, 0x6a46020, 0xc422fc9c60, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc9c00, 0x6a551e0, 0xc42121b340, 0xc423200db0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4201cb440, 0x6a611e0, 0xc421f06a80, 0x0, 0x0, 0xc42248100d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4201cb440, 0x6a611e0, 0xc421f06a80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420fb1140, 0x6a61a80, 0xc4221c81e0, 0xc4222c7d30, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4221c81e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc4221c81e0, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1903 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc420fae6c0, 0xc4202d8b90, 0x7, 0xc4241ee640, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc420fae6c0, 0x7, 0xc4241ee640, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc420fae6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8d200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc420fae6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8d200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc420fae6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8d200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc420fae6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8d100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2d180, 0x6a552a0, 0xc420fae660, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8d100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc422cefac0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8d100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc42271da00, 0x6a551e0, 0xc422cefac0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc422cefac0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc422cefac0, 0xc42172b878, 0xc42235a700, 0xda, 0xda) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc42271da00, 0x6a551e0, 0xc422cefac0, 0xc42235a700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc421b273d0, 0x6a551e0, 0xc422cefac0, 0xc4226834e0, 0x12, 0x20, 0xcb1, 0x6a651a0, 0xc4223adb00, 0xc4217d5ce0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc421064d80, 0x6a551e0, 0xc422cefac0, 0xc421b27440, 0xc421b273d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc421064d80, 0xc4223ed828, 0xc42172cb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc4241d2c00, 0x5c46757, 0xc, 0x6a4c120, 0xc421064d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc4241d2c00, 0x6a4c120, 0xc421064d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc422cefac0, 0x6a4c120, 0xc421064d80, 0x5d28ea0, 0x0, 0x0, 0xc421064db0, 0xc4226972c0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc422cefac0, 0xc421b27440, 0xc421b273d0, 0x6a4c120, 0xc421064d80, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc422cefac0, 0xc421b27440, 0xc421b273d0, 0x6a4c120, 0xc421064d80, 0x0, 0xc4226972c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc421b27648, 0x6a551e0, 0xc422cefac0, 0xc421b27440, 0xc421b273d0, 0xaa8f50768f33700f, 0xc42172d590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc421b27000, 0x6a551e0, 0xc422cefac0, 0xc421b273d0, 0x1, 0x73e91b0, 0xc423934c60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc421b27000, 0x6a551e0, 0xc422cefac0, 0x6a568e0, 0xc422324cc0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc421b27000, 0x6a551e0, 0xc422cefac0, 0x6a568e0, 0xc422324cc0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421b27000, 0x6a551e0, 0xc422cefac0, 0x6a568e0, 0xc422324cc0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421b27000, 0x6a551e0, 0xc423228c40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc423228c40, 0x0, 0x0, 0xc421309035, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423200440, 0xc4204a4700, 0x6a551e0, 0xc423228c40, 0xc42117ca80, 0x5400, 0x15000, 0xc420724a90, 0xc422f7a8c0, 0xc423200430) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1567 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc4219f7890, 0xc4202d8b90, 0x7, 0xc4218dd900, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc4219f7890, 0x7, 0xc4218dd900, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc4219f7890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4211ef500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc4219f7890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4211ef500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc4219f7890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4211ef500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc4219f7890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4211ef400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42117d880, 0x6a552a0, 0xc4219f7800, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4211ef400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc424909540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4211ef400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423070000, 0x6a551e0, 0xc424909540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc423070000, 0x6a551e0, 0xc424909540, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc423070000, 0x6a551e0, 0xc424909540, 0xc423070000, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc423070000, 0x6a551e0, 0xc424909540, 0xc4247d5bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc420b25000, 0x6a551e0, 0xc424909540, 0x6a568e0, 0xc4226baf60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc420b25000, 0x6a551e0, 0xc424909540, 0x6a568e0, 0xc4226baf60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc420b25000, 0x6a551e0, 0xc4212f04c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc4212f04c0, 0x0, 0x0, 0xc421b0d00d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc421062f00, 0xc4204a4700, 0x6a551e0, 0xc4212f04c0, 0xc4224d4540, 0x5400, 0x15000, 0xc420724a90, 0xc421b22500, 0xc421062ef0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115255 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423875680, 0x45563f3, 0xc424ec11e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423875680, 0x5d2c558, 0xc424ec1420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc424596240, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc424596240, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b311f8, 0x5b6ec40, 0xc422cd6960, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423a47c00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b311f8, 0x5b6ec40, 0xc422cd6960, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc423a47c00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b311f8, 0x5b6ec40, 0xc422cd6960, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423a47c00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b311f8, 0x5b6ec40, 0xc422cd6960, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80830, 0x6a551e0, 0xc423a47c00, 0xc423b311f8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x80d215e43) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e75c00, 0xc423b311e0, 0x6a551e0, 0xc423a47c00, 0xc400000008, 0xc422a5b540, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e75c00, 0x6a551e0, 0xc423a47c00, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e75c00, 0x6a551e0, 0xc423a47c00, 0xc423b30fd0, 0xc422cd6780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 1724 [IO wait]: internal/poll.runtime_pollWait(0x73ea168, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb518, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb518, 0xc4220ccc00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb500, 0xc4220ccc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb500, 0xc4220ccc00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f4a8, 0xc4220ccc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f7f2c0, 0xc4220ccc00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421012390, 0x73ccbf8, 0xc421f7f2c0, 0x5, 0xc421f7f2c0, 0xba) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f07880, 0x5d2d117, 0xc421f079a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f07880, 0xc423017000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421fd6a80, 0xc423017000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4232265a0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4232265a0, 0xbe9de8a323f5bfe0, 0x7c44f1c85, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4232266d0, 0x6a46020, 0xc4232265a0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc423226540, 0x6a551e0, 0xc42132bb80, 0xc4232014e0, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4203b4bc0, 0x6a611e0, 0xc421f07880, 0x0, 0x0, 0xc421870022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc4203b4bc0, 0x6a611e0, 0xc421f07880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222cd9e0, 0x6a61a80, 0xc421f7f2c0, 0xc422c11930, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f7f2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc421f7f2c0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1651 [IO wait]: internal/poll.runtime_pollWait(0x73e4790, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb118, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb118, 0xc421b0e000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb100, 0xc421b0e000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb100, 0xc421b0e000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f228, 0xc421b0e000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f7ebe0, 0xc421b0e000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420d691a0, 0x73ccbf8, 0xc421f7ebe0, 0x5, 0xc421f7ebe0, 0xb6) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b16000, 0x5d2d117, 0xc420b16120, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b16000, 0xc421b3c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421f74160, 0xc421b3c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4224d45a0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4224d45a0, 0xbe9de8a314adaf98, 0x7b507276f, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4224d46d0, 0x6a46020, 0xc4224d45a0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4224d4540, 0x6a551e0, 0xc4212f0480, 0xc421062eb0, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420fee500, 0x6a611e0, 0xc420b16000, 0x0, 0x0, 0xc421b0d00d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc420fee500, 0x6a611e0, 0xc420b16000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420d69170, 0x6a61a80, 0xc421f7ebe0, 0xc422571fc0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f7ebe0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc421f7ebe0, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1654 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724b40, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a97b98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a97b98, 0xc422916000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a97b80, 0xc422916000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a97b80, 0xc422916000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f82f8, 0xc422916000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420732bd0, 0x77aa1a0, 0xc4204f82f8, 0x5, 0xc4204f82f8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206a7180, 0x5d2d117, 0xc4206a72a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206a7180, 0xc4211dd000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ac960, 0xc420cb3ba0, 0x5, 0x200, 0xc422a156e8, 0x6, 0xc4206a7350) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ac960, 0xc420cb3ba0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422f85aa0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ac960, 0xc420cb3ba0, 0x5, 0x200, 0x77c3118, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420cb3b80, 0xc422f85aa0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420cb3b80, 0xc422f85aa0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420cb3b80, 0xc422a15890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420cb3b80, 0xc42376abe0, 0x4b, 0xc422a15960, 0xc422a15910, 0x4047467, 0x0, 0x5a02440, 0xc422a15901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420cb3b80, 0xc42376abe0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420cb3b80, 0x6a55220, 0xc4200180f8, 0xc42376abe0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420cb3b80, 0xc42376abe0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4211f1b20, 0xc422a15c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4211f1b20, 0xc422a15d48, 0xc42376abe0, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc42376abe0, 0x4b, 0x0, 0x0, 0x0, 0xc420034701, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc42376abe0, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc42376abe0, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc422f85a80, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x0, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1727 [IO wait]: internal/poll.runtime_pollWait(0x73e3ad0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb498, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb498, 0xc4220cd000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb480, 0xc4220cd000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb480, 0xc4220cd000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f408, 0xc4220cd000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f7f040, 0xc4220cd000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210123c0, 0x73ccbf8, 0xc421f7f040, 0x5, 0xc421f7f040, 0xbb) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f07c00, 0x5d2d117, 0xc421f07d20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f07c00, 0xc422ff9000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422eb3a40, 0xc422ff9000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fd2ae0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fd2ae0, 0xbe9de8a30da6fec0, 0x7ae00873f, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fd2c10, 0x6a46020, 0xc422fd2ae0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fd2a80, 0x6a551e0, 0xc423228e40, 0xc423200380, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4203b5580, 0x6a611e0, 0xc421f07c00, 0x0, 0x0, 0xc421871037, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc4203b5580, 0x6a611e0, 0xc421f07c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4228ba750, 0x6a61a80, 0xc421f7f040, 0xc422c1ced0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f7f040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc421f7f040, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1648 [IO wait]: internal/poll.runtime_pollWait(0x73e9e68, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb798, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb798, 0xc421fd0800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb780, 0xc421fd0800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb780, 0xc421fd0800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b02b8, 0xc421fd0800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f9a000, 0xc421fd0800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4228ba810, 0x73ccbf8, 0xc421f9a000, 0x5, 0xc421f9a000, 0xc1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42251d180, 0x5d2d117, 0xc42251d2a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42251d180, 0xc4221fa000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422697760, 0xc4221fa000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42117d020) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42117d020, 0xbe9de8a31fd1fb18, 0x7c02b5f83, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42117d150, 0x6a46020, 0xc42117d020, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42117cfc0, 0x6a551e0, 0xc420b4d1c0, 0xc42221c7d0, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42121a4c0, 0x6a611e0, 0xc42251d180, 0x0, 0x0, 0xc422c3504a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc42121a4c0, 0x6a611e0, 0xc42251d180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4228ba7e0, 0x6a61a80, 0xc421f9a000, 0xc422c1cf00, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f9a000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc421f9a000, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1655 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x938f558, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a97f98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a97f98, 0xc421b35400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a97f80, 0xc421b35400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a97f80, 0xc421b35400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f82a8, 0xc421b35400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4204d8360, 0x77aa1a0, 0xc4204f82a8, 0x5, 0xc4204f82a8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42204c000, 0x5d2d117, 0xc42204c120, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42204c000, 0xc421b56000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc421fb6060, 0xc421138860, 0x5, 0x200, 0xc422aaf6e8, 0x6, 0xc42204c1d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc421fb6060, 0xc421138860, 0x5, 0x200, 0x5, 0x40147f7, 0xc421ebea20, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc421fb6060, 0xc421138860, 0x5, 0x200, 0x9391c20, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc421138840, 0xc421ebea20, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc421138840, 0xc421ebea20, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc421138840, 0xc422aaf890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc421138840, 0xc4228bc460, 0x4b, 0xc422aaf960, 0xc422aaf910, 0x4047467, 0x0, 0x5a02440, 0xc422aaf901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc421138840, 0xc4228bc460, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc421138840, 0x6a55220, 0xc4200180f8, 0xc4228bc460, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc421138840, 0xc4228bc460, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420b8a850, 0xc422aafc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc420b8a850, 0xc422aafd48, 0xc4228bc460, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4228bc460, 0x4b, 0x0, 0x0, 0x0, 0xc421945801, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4228bc460, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc4228bc460, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc421ebe9a0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x1, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1657 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724fc0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96f98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96f98, 0xc422917000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96f80, 0xc422917000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96f80, 0xc422917000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b04b0, 0xc422917000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420732e10, 0x77aa1a0, 0xc4205b04b0, 0x5, 0xc4205b04b0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206a7880, 0x5d2d117, 0xc4206a79a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206a7880, 0xc42291d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228aca20, 0xc42134d8e0, 0x5, 0x200, 0xc4225556e8, 0x6, 0xc4206a7a50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228aca20, 0xc42134d8e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422b07fe0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228aca20, 0xc42134d8e0, 0x5, 0x200, 0x9392210, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc42134d8c0, 0xc422b07fe0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc42134d8c0, 0xc422b07fe0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc42134d8c0, 0xc422555890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc42134d8c0, 0xc4218de370, 0x4b, 0xc422555960, 0xc422555910, 0x4047467, 0x0, 0x5a02440, 0xc422555901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc42134d8c0, 0xc4218de370, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc42134d8c0, 0x6a55220, 0xc4200180f8, 0xc4218de370, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc42134d8c0, 0xc4218de370, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4223cf3b0, 0xc422555c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4223cf3b0, 0xc422555d48, 0xc4218de370, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4218de370, 0x4b, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4218de370, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4218de370, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc422b07fc0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x3, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1658 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724c00, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a97998, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a97998, 0xc4216d3000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a97980, 0xc4216d3000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a97980, 0xc4216d3000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8320, 0xc4216d3000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc42086ef30, 0x77aa1a0, 0xc4204f8320, 0x5, 0xc4204f8320, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfa700, 0x5d2d117, 0xc421bfa820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfa700, 0xc4229c1000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308900, 0xc420cb3620, 0x5, 0x200, 0xc422f1f6e8, 0x6, 0xc421bfa8d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308900, 0xc420cb3620, 0x5, 0x200, 0x5, 0x40147f7, 0xc4215e2240, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308900, 0xc420cb3620, 0x5, 0x200, 0x73ee840, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420cb3600, 0xc4215e2240, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420cb3600, 0xc4215e2240, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420cb3600, 0xc422f1f890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420cb3600, 0xc421876050, 0x4b, 0xc422f1f960, 0xc422f1f910, 0x4047467, 0x0, 0x5a02440, 0xc422f1f901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420cb3600, 0xc421876050, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420cb3600, 0x6a55220, 0xc4200180f8, 0xc421876050, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420cb3600, 0xc421876050, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4223073b0, 0xc422f1fc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4223073b0, 0xc422f1fd48, 0xc421876050, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc421876050, 0x4b, 0x0, 0x0, 0x0, 0xc4202cce01, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc421876050, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc421876050, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc420382bc0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x4, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1659 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e9aa8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420462318, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420462318, 0xc421054000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420462300, 0xc421054000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420462300, 0xc421054000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f81c0, 0xc421054000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc421013cb0, 0x77aa1a0, 0xc4204f81c0, 0x5, 0xc4204f81c0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfb880, 0x5d2d117, 0xc422bfb9a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfb880, 0xc421052000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ac540, 0xc422b6c5a0, 0x5, 0x200, 0xc422f236e8, 0x6, 0xc422bfba50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ac540, 0xc422b6c5a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422c2c0e0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ac540, 0xc422b6c5a0, 0x5, 0x200, 0x77b4618, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6c580, 0xc422c2c0e0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6c580, 0xc422c2c0e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6c580, 0xc422f23890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6c580, 0xc4226c6230, 0x4b, 0xc422f23960, 0xc422f23910, 0x4047467, 0x0, 0x5a02440, 0xc422f23901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6c580, 0xc4226c6230, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6c580, 0x6a55220, 0xc4200180f8, 0xc4226c6230, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6c580, 0xc4226c6230, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc421108000, 0xc422f23c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc421108000, 0xc422f23d48, 0xc4226c6230, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4226c6230, 0x4b, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4226c6230, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4226c6230, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc422c2c0c0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x5, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1660 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eae68, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421970418, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421970418, 0xc4224dbc00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421970400, 0xc4224dbc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421970400, 0xc4224dbc00, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e590, 0xc4224dbc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2fe30, 0x77aa1a0, 0xc42000e590, 0x5, 0xc42000e590, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42200a000, 0x5d2d117, 0xc42200a120, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42200a000, 0xc421e91000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4223084e0, 0xc422abcb20, 0x5, 0x200, 0xc422ea56e8, 0x6, 0xc42200a1d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4223084e0, 0xc422abcb20, 0x5, 0x200, 0x5, 0x40147f7, 0xc4226979c0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4223084e0, 0xc422abcb20, 0x5, 0x200, 0x938bfb0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422abcb00, 0xc4226979c0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422abcb00, 0xc4226979c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422abcb00, 0xc422ea5890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422abcb00, 0xc4228bd950, 0x4b, 0xc422ea5960, 0xc422ea5910, 0x4047467, 0x0, 0x5a02440, 0xc422ea5901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422abcb00, 0xc4228bd950, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422abcb00, 0x6a55220, 0xc4200180f8, 0xc4228bd950, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422abcb00, 0xc4228bd950, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc422528a10, 0xc422ea5c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422528a10, 0xc422ea5d48, 0xc4228bd950, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4228bd950, 0x4b, 0x0, 0x0, 0x0, 0xc420034701, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4228bd950, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4228bd950, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc4226979a0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x6, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1661 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724e40, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a97398, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a97398, 0xc4216d3800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a97380, 0xc4216d3800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a97380, 0xc4216d3800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f83a8, 0xc4216d3800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc42086eff0, 0x77aa1a0, 0xc4204f83a8, 0x5, 0xc4204f83a8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfae00, 0x5d2d117, 0xc421bfaf20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfae00, 0xc42291e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308960, 0xc420cb25a0, 0x5, 0x200, 0xc42243f6e8, 0x6, 0xc421bfafd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308960, 0xc420cb25a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc421209e20, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308960, 0xc420cb25a0, 0x5, 0x200, 0x7712bf0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420cb2580, 0xc421209e20, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420cb2580, 0xc421209e20, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420cb2580, 0xc42243f890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420cb2580, 0xc422e2ac30, 0x4b, 0xc42243f960, 0xc42243f910, 0x4047467, 0x0, 0x5a02440, 0xc42243f901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420cb2580, 0xc422e2ac30, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420cb2580, 0x6a55220, 0xc4200180f8, 0xc422e2ac30, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420cb2580, 0xc422e2ac30, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420f6bea0, 0xc42243fc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc420f6bea0, 0xc42243fd48, 0xc422e2ac30, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422e2ac30, 0x4b, 0x0, 0x0, 0x0, 0xc420034701, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422e2ac30, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc422e2ac30, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc421209da0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x7, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1662 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724cc0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a97798, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a97798, 0xc42203a800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a97780, 0xc42203a800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a97780, 0xc42203a800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8348, 0xc42203a800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420733f50, 0x77aa1a0, 0xc4204f8348, 0x5, 0xc4204f8348, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42232e380, 0x5d2d117, 0xc42232e4a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42232e380, 0xc42265d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ad260, 0xc420cb30a0, 0x5, 0x200, 0xc4224436e8, 0x6, 0xc42232e550) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ad260, 0xc420cb30a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc4222ef120, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ad260, 0xc420cb30a0, 0x5, 0x200, 0x77b6010, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420cb3080, 0xc4222ef120, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420cb3080, 0xc4222ef120, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420cb3080, 0xc422443890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420cb3080, 0xc422292140, 0x4b, 0xc422443960, 0xc422443910, 0x4047467, 0x0, 0x5a02440, 0xc422443901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420cb3080, 0xc422292140, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420cb3080, 0x6a55220, 0xc4200180f8, 0xc422292140, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420cb3080, 0xc422292140, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4211f11f0, 0xc422443c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4211f11f0, 0xc422443d48, 0xc422292140, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422292140, 0x4b, 0x0, 0x0, 0x0, 0xc421945801, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422292140, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc422292140, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc4222ef100, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x8, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1663 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e99e8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cc998, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cc998, 0xc421ee7800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cc980, 0xc421ee7800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cc980, 0xc421ee7800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8198, 0xc421ee7800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420162c60, 0x77aa1a0, 0xc4204f8198, 0x5, 0xc4204f8198, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42200b880, 0x5d2d117, 0xc42200b9a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42200b880, 0xc421c52000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308600, 0xc420f98020, 0x5, 0x200, 0xc422d116e8, 0x6, 0xc42200ba50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308600, 0xc420f98020, 0x5, 0x200, 0x5, 0x40147f7, 0xc4211d9800, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308600, 0xc420f98020, 0x5, 0x200, 0x73d4cf8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420f98000, 0xc4211d9800, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420f98000, 0xc4211d9800, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420f98000, 0xc422d11890, 0x4c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420f98000, 0xc42357d0e0, 0x4b, 0xc422d11960, 0xc422d11910, 0x4047467, 0x0, 0x5a02440, 0xc422d11901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420f98000, 0xc42357d0e0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420f98000, 0x6a55220, 0xc4200180f8, 0xc42357d0e0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420f98000, 0xc42357d0e0, 0x4b, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4211f1110, 0xc422d11c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4211f1110, 0xc422d11d48, 0xc42357d0e0, 0x4b, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc42357d0e0, 0x4b, 0x0, 0x0, 0x0, 0xc4206f9801, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc42357d0e0, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0x4b, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc42357d0e0, 0x4b, 0x0, 0x0, 0x0, 0x4b, 0xc4211d97e0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x9, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1664 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e40d0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cce18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cce18, 0xc422af4000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cce00, 0xc422af4000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cce00, 0xc422af4000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef250, 0xc422af4000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4222cc810, 0x77aa1a0, 0xc4206ef250, 0x5, 0xc4206ef250, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42251dc00, 0x5d2d117, 0xc42251dd20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42251dc00, 0xc422001000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422980b40, 0xc422b6c860, 0x5, 0x200, 0x40, 0x3f, 0xc4215db6c0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422980b40, 0xc422b6c860, 0x5, 0x200, 0x5, 0x40147f7, 0xc422d4c7e0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422980b40, 0xc422b6c860, 0x5, 0x200, 0x73e0490, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6c840, 0xc422d4c7e0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6c840, 0xc422d4c7e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6c840, 0xc4215db890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6c840, 0xc4218defa0, 0x4c, 0xc4215db960, 0xc4215db910, 0x4047467, 0x0, 0x5a02440, 0xc4215db901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6c840, 0xc4218defa0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6c840, 0x6a55220, 0xc4200180f8, 0xc4218defa0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6c840, 0xc4218defa0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420269ea0, 0xc4215dbc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc420269ea0, 0xc4215dbd48, 0xc4218defa0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4218defa0, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4218defa0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc4218defa0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422d4c7c0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0xa, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1665 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7725140, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96a98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96a98, 0xc42203b000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96a80, 0xc42203b000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96a80, 0xc42203b000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0508, 0xc42203b000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420cee000, 0x77aa1a0, 0xc4205b0508, 0x5, 0xc4205b0508, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42232e700, 0x5d2d117, 0xc42232e820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42232e700, 0xc421b50000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ad2c0, 0xc42134cde0, 0x5, 0x200, 0xc4215df6e8, 0x6, 0xc42232e8d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ad2c0, 0xc42134cde0, 0x5, 0x200, 0x5, 0x40147f7, 0xc420349260, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ad2c0, 0xc42134cde0, 0x5, 0x200, 0x73c4c60, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc42134cdc0, 0xc420349260, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc42134cdc0, 0xc420349260, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc42134cdc0, 0xc4215df890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc42134cdc0, 0xc4220cff90, 0x4c, 0xc4215df960, 0xc4215df910, 0x4047467, 0x0, 0x5a02440, 0xc4215df901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc42134cdc0, 0xc4220cff90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc42134cdc0, 0x6a55220, 0xc4200180f8, 0xc4220cff90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc42134cdc0, 0xc4220cff90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420e75ea0, 0xc4215dfc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420e75ea0, 0xc4215dfd48, 0xc4220cff90, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4220cff90, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4220cff90, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4220cff90, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc420349240, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0xb, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1666 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7725200, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96898, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96898, 0xc422917800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96880, 0xc422917800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96880, 0xc422917800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee5c8, 0xc422917800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420732e70, 0x77aa1a0, 0xc4206ee5c8, 0x5, 0xc4206ee5c8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206a7c00, 0x5d2d117, 0xc4206a7d20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206a7c00, 0xc42265c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228aca80, 0xc42134c860, 0x5, 0x200, 0xc421f416e8, 0x6, 0xc4206a7dd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228aca80, 0xc42134c860, 0x5, 0x200, 0x5, 0x40147f7, 0xc422b19740, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228aca80, 0xc42134c860, 0x5, 0x200, 0x77c4360, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc42134c840, 0xc422b19740, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc42134c840, 0xc422b19740, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc42134c840, 0xc421f41890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc42134c840, 0xc421f7c500, 0x4c, 0xc421f41960, 0xc421f41910, 0x4047467, 0x0, 0x5a02440, 0xc421f41901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc42134c840, 0xc421f7c500, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc42134c840, 0x6a55220, 0xc4200180f8, 0xc421f7c500, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc42134c840, 0xc421f7c500, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc422326e70, 0xc421f41c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422326e70, 0xc421f41d48, 0xc421f7c500, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc421f7c500, 0x4c, 0x0, 0x0, 0x0, 0xc420064301, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc421f7c500, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc421f7c500, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422b19720, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0xc, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1667 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e9da8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc423a3c118, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc423a3c118, 0xc420c73400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc423a3c100, 0xc420c73400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc423a3c100, 0xc420c73400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc423ba77e8, 0xc420c73400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc423890a20, 0x77aa1a0, 0xc423ba77e8, 0x5, 0xc423ba77e8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421944a80, 0x5d2d117, 0xc421944ba0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421944a80, 0xc4224e8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4238c1620, 0xc422c21620, 0x5, 0x200, 0xc420d056e8, 0x6, 0xc421944c50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4238c1620, 0xc422c21620, 0x5, 0x200, 0x5, 0x40147f7, 0xc421c2cf20, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4238c1620, 0xc422c21620, 0x5, 0x200, 0x73dee00, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422c21600, 0xc421c2cf20, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422c21600, 0xc421c2cf20, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422c21600, 0xc420d05890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422c21600, 0xc4228bda40, 0x4c, 0xc420d05960, 0xc420d05910, 0x4047467, 0x0, 0x5a02440, 0xc420d05901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422c21600, 0xc4228bda40, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422c21600, 0x6a55220, 0xc4200180f8, 0xc4228bda40, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422c21600, 0xc4228bda40, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc423f50e70, 0xc420d05c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc423f50e70, 0xc420d05d48, 0xc4228bda40, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4228bda40, 0x4c, 0x0, 0x0, 0x0, 0xc420064301, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4228bda40, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc4228bda40, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc421c2cee0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0xd, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1668 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73ea868, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc422d9fb18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc422d9fb18, 0xc422523c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc422d9fb00, 0xc422523c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc422d9fb00, 0xc422523c00, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206eedc8, 0xc422523c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4211e83c0, 0x77aa1a0, 0xc4206eedc8, 0x5, 0xc4206eedc8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4247e8700, 0x5d2d117, 0xc4247e8820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4247e8700, 0xc4216ba000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420efbaa0, 0xc422b610a0, 0x5, 0x200, 0xc424b316e8, 0x6, 0xc4247e88d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420efbaa0, 0xc422b610a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422696200, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420efbaa0, 0xc422b610a0, 0x5, 0x200, 0x938bfb0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b61080, 0xc422696200, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b61080, 0xc422696200, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b61080, 0xc424b31890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b61080, 0xc42551c370, 0x4c, 0xc424b31960, 0xc424b31910, 0x4047467, 0x0, 0x5a02440, 0xc424b31901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b61080, 0xc42551c370, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b61080, 0x6a55220, 0xc4200180f8, 0xc42551c370, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b61080, 0xc42551c370, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4225eb9d0, 0xc424b31c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4225eb9d0, 0xc424b31d48, 0xc42551c370, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42551c370, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42551c370, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc42551c370, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4226961e0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0xe, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1669 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x77255c0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc423641798, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc423641798, 0xc422569800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc423641780, 0xc422569800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc423641780, 0xc422569800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206eec58, 0xc422569800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc421ab2660, 0x77aa1a0, 0xc4206eec58, 0x5, 0xc4206eec58, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224ec700, 0x5d2d117, 0xc4224ec820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224ec700, 0xc423e4c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc423141620, 0xc420f98b20, 0x5, 0x200, 0xc42179f6e8, 0x6, 0xc4224ec8d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc423141620, 0xc420f98b20, 0x5, 0x200, 0x5, 0x40147f7, 0xc422e27fe0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc423141620, 0xc420f98b20, 0x5, 0x200, 0x9396ea8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420f98b00, 0xc422e27fe0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420f98b00, 0xc422e27fe0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420f98b00, 0xc42179f890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420f98b00, 0xc422e2b130, 0x4c, 0xc42179f960, 0xc42179f910, 0x4047467, 0x0, 0x5a02440, 0xc42179f901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420f98b00, 0xc422e2b130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420f98b00, 0x6a55220, 0xc4200180f8, 0xc422e2b130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420f98b00, 0xc422e2b130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420b8b570, 0xc42179fc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc420b8b570, 0xc42179fd48, 0xc422e2b130, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e2b130, 0x4c, 0x0, 0x0, 0x0, 0xc420034701, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e2b130, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc422e2b130, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422e27fc0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0xf, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1670 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e9928, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420462598, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420462598, 0xc422b78400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420462580, 0xc422b78400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420462580, 0xc422b78400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8168, 0xc422b78400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2fb00, 0x77aa1a0, 0xc4204f8168, 0x5, 0xc4204f8168, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422526000, 0x5d2d117, 0xc422526120, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422526000, 0xc422abe000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308240, 0xc422b6d0a0, 0x5, 0x200, 0xc4228fb6e8, 0x6, 0xc4225261d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308240, 0xc422b6d0a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422e26220, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308240, 0xc422b6d0a0, 0x5, 0x200, 0x9396ea8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6d080, 0xc422e26220, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6d080, 0xc422e26220, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6d080, 0xc4228fb890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6d080, 0xc423689f90, 0x4c, 0xc4228fb960, 0xc4228fb910, 0x4047467, 0x0, 0x5a02440, 0xc4228fb901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6d080, 0xc423689f90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6d080, 0x6a55220, 0xc4200180f8, 0xc423689f90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6d080, 0xc423689f90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc422529340, 0xc4228fbc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422529340, 0xc4228fbd48, 0xc423689f90, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc423689f90, 0x4c, 0x0, 0x0, 0x0, 0xc420034701, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc423689f90, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc423689f90, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422e261e0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x10, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1671 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eac28, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421970c18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421970c18, 0xc421694800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421970c00, 0xc421694800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421970c00, 0xc421694800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8238, 0xc421694800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420b7e780, 0x77aa1a0, 0xc4204f8238, 0x5, 0xc4204f8238, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfb500, 0x5d2d117, 0xc421bfb620, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfb500, 0xc421947000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308de0, 0xc4204c62e0, 0x5, 0x200, 0xc421c216e8, 0x6, 0xc421bfb6d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308de0, 0xc4204c62e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc4223d6900, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308de0, 0xc4204c62e0, 0x5, 0x200, 0x77b1e50, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc4204c62c0, 0xc4223d6900, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc4204c62c0, 0xc4223d6900, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc4204c62c0, 0xc421c21890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc4204c62c0, 0xc42376a190, 0x4c, 0xc421c21960, 0xc421c21910, 0x4047467, 0x0, 0x5a02440, 0xc421c21901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc4204c62c0, 0xc42376a190, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc4204c62c0, 0x6a55220, 0xc4200180f8, 0xc42376a190, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc4204c62c0, 0xc42376a190, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420b8a8c0, 0xc421c21c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420b8a8c0, 0xc421c21d48, 0xc42376a190, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42376a190, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42376a190, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc42376a190, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4223d68e0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x11, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1672 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e3f50, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cd018, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cd018, 0xc422668800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cd000, 0xc422668800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cd000, 0xc422668800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef228, 0xc422668800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4201631d0, 0x77aa1a0, 0xc4206ef228, 0x5, 0xc4206ef228, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfa380, 0x5d2d117, 0xc421bfa4a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfa380, 0xc4229c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4223086c0, 0xc422b6cde0, 0x5, 0x200, 0xc421faf6e8, 0x6, 0xc421bfa550) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4223086c0, 0xc422b6cde0, 0x5, 0x200, 0x5, 0x40147f7, 0xc4223d68e0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4223086c0, 0xc422b6cde0, 0x5, 0x200, 0x77b1e50, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6cdc0, 0xc4223d68e0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6cdc0, 0xc4223d68e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6cdc0, 0xc421faf890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6cdc0, 0xc4231db2c0, 0x4c, 0xc421faf960, 0xc421faf910, 0x4047467, 0x0, 0x5a02440, 0xc421faf901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6cdc0, 0xc4231db2c0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6cdc0, 0x6a55220, 0xc4200180f8, 0xc4231db2c0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6cdc0, 0xc4231db2c0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc421109420, 0xc421fafc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc421109420, 0xc421fafd48, 0xc4231db2c0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4231db2c0, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4231db2c0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4231db2c0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4223d6860, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x12, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1673 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eace8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421970818, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421970818, 0xc422b78c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421970800, 0xc422b78c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421970800, 0xc422b78c00, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e538, 0xc422b78c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2fb90, 0x77aa1a0, 0xc42000e538, 0x5, 0xc42000e538, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422526380, 0x5d2d117, 0xc4225264a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422526380, 0xc422abf000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4223082a0, 0xc420290020, 0x5, 0x200, 0xc422e1f6e8, 0x6, 0xc422526550) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4223082a0, 0xc420290020, 0x5, 0x200, 0x5, 0x40147f7, 0xc422e262e0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4223082a0, 0xc420290020, 0x5, 0x200, 0x9396ea8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420290000, 0xc422e262e0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420290000, 0xc422e262e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420290000, 0xc422e1f890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420290000, 0xc4228bc410, 0x4c, 0xc422e1f960, 0xc422e1f910, 0x4047467, 0x0, 0x5a02440, 0xc422e1f901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420290000, 0xc4228bc410, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420290000, 0x6a55220, 0xc4200180f8, 0xc4228bc410, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420290000, 0xc4228bc410, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4211f1180, 0xc422e1fc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4211f1180, 0xc422e1fd48, 0xc4228bc410, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4228bc410, 0x4c, 0x0, 0x0, 0x0, 0xc4206f9801, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4228bc410, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc4228bc410, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422e262a0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x13, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1674 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eada8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421970618, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421970618, 0xc420cf4400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421970600, 0xc420cf4400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421970600, 0xc420cf4400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e568, 0xc420cf4400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4204d9e60, 0x77aa1a0, 0xc42000e568, 0x5, 0xc42000e568, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42204ce00, 0x5d2d117, 0xc42204cf20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42204ce00, 0xc420cf2000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc421fb6780, 0xc422abd620, 0x5, 0x200, 0xc422e236e8, 0x6, 0xc42204cfd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc421fb6780, 0xc422abd620, 0x5, 0x200, 0x5, 0x40147f7, 0xc4215e26a0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc421fb6780, 0xc422abd620, 0x5, 0x200, 0x73ee840, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422abd600, 0xc4215e26a0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422abd600, 0xc4215e26a0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422abd600, 0xc422e23890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422abd600, 0xc4232cc870, 0x4c, 0xc422e23960, 0xc422e23910, 0x4047467, 0x0, 0x5a02440, 0xc422e23901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422abd600, 0xc4232cc870, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422abd600, 0x6a55220, 0xc4200180f8, 0xc4232cc870, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422abd600, 0xc4232cc870, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420f6ba40, 0xc422e23c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420f6ba40, 0xc422e23d48, 0xc4232cc870, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4232cc870, 0x4c, 0x0, 0x0, 0x0, 0xc4206f9801, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4232cc870, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4232cc870, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4215e2680, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x14, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1675 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73ea9e8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228ccd18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228ccd18, 0xc423532c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228ccd00, 0xc423532c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228ccd00, 0xc423532c00, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4241caf00, 0xc423532c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc422d98270, 0x77aa1a0, 0xc4241caf00, 0x5, 0xc4241caf00, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42271f880, 0x5d2d117, 0xc42271f9a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42271f880, 0xc423e56000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422417e00, 0xc420f98de0, 0x5, 0x200, 0xc421a116e8, 0x6, 0xc42271fa50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422417e00, 0xc420f98de0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422eb23c0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422417e00, 0xc420f98de0, 0x5, 0x200, 0x77c0820, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420f98dc0, 0xc422eb23c0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420f98dc0, 0xc422eb23c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420f98dc0, 0xc421a11890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420f98dc0, 0xc422e2a0f0, 0x4c, 0xc421a11960, 0xc421a11910, 0x4047467, 0x0, 0x5a02440, 0xc421a11901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420f98dc0, 0xc422e2a0f0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420f98dc0, 0x6a55220, 0xc4200180f8, 0xc422e2a0f0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420f98dc0, 0xc422e2a0f0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc423a97110, 0xc421a11c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc423a97110, 0xc421a11d48, 0xc422e2a0f0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e2a0f0, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e2a0f0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc422e2a0f0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422eb23a0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x15, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1676 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eaf28, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421970218, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421970218, 0xc42228d000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421970200, 0xc42228d000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421970200, 0xc42228d000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee2f0, 0xc42228d000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2fbf0, 0x77aa1a0, 0xc4206ee2f0, 0x5, 0xc4206ee2f0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422526700, 0x5d2d117, 0xc422526820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422526700, 0xc421f68000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308300, 0xc422abc5a0, 0x5, 0x200, 0xc421a156e8, 0x6, 0xc4225268d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308300, 0xc422abc5a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc42230aac0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308300, 0xc422abc5a0, 0x5, 0x200, 0x77ae3d0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422abc580, 0xc42230aac0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422abc580, 0xc42230aac0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422abc580, 0xc421a15890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422abc580, 0xc423e92e60, 0x4c, 0xc421a15960, 0xc421a15910, 0x4047467, 0x0, 0x5a02440, 0xc421a15901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422abc580, 0xc423e92e60, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422abc580, 0x6a55220, 0xc4200180f8, 0xc423e92e60, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422abc580, 0xc423e92e60, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4211f0d90, 0xc421a15c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4211f0d90, 0xc421a15d48, 0xc423e92e60, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc423e92e60, 0x4c, 0x0, 0x0, 0x0, 0xc420466001, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc423e92e60, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc423e92e60, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc42230aaa0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x16, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1677 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eafe8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421970018, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421970018, 0xc421054800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421970000, 0xc421054800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421970000, 0xc421054800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee318, 0xc421054800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc421013e00, 0x77aa1a0, 0xc4206ee318, 0x5, 0xc4206ee318, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfbc00, 0x5d2d117, 0xc422bfbd20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfbc00, 0xc421053000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ac5a0, 0xc422b618e0, 0x5, 0x200, 0xc4220776e8, 0x6, 0xc422bfbdd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ac5a0, 0xc422b618e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422c2c020, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ac5a0, 0xc422b618e0, 0x5, 0x200, 0x77b4618, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b618c0, 0xc422c2c020, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b618c0, 0xc422c2c020, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b618c0, 0xc422077890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b618c0, 0xc4226c6280, 0x4c, 0xc422077960, 0xc422077910, 0x4047467, 0x0, 0x5a02440, 0xc422077901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b618c0, 0xc4226c6280, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b618c0, 0x6a55220, 0xc4200180f8, 0xc4226c6280, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b618c0, 0xc4226c6280, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420e756c0, 0xc422077c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420e756c0, 0xc422077d48, 0xc4226c6280, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4226c6280, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4226c6280, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4226c6280, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422c2c000, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x17, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1678 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eb468, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cde18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cde18, 0xc42228fc00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cde00, 0xc42228fc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cde00, 0xc42228fc00, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e028, 0xc42228fc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4204e11a0, 0x77aa1a0, 0xc42000e028, 0x5, 0xc42000e028, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421a0dc00, 0x5d2d117, 0xc421a0dd20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421a0dc00, 0xc4224d7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422981d40, 0xc422b61360, 0x5, 0x200, 0xc42207b6e8, 0x6, 0xc421a0ddd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422981d40, 0xc422b61360, 0x5, 0x200, 0x5, 0x40147f7, 0xc424bd6ee0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422981d40, 0xc422b61360, 0x5, 0x200, 0xca95d30, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b61340, 0xc424bd6ee0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b61340, 0xc424bd6ee0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b61340, 0xc42207b890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b61340, 0xc4230c0e60, 0x4c, 0xc42207b960, 0xc42207b910, 0x4047467, 0x0, 0x5a02440, 0xc42207b901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b61340, 0xc4230c0e60, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b61340, 0x6a55220, 0xc4200180f8, 0xc4230c0e60, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b61340, 0xc4230c0e60, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4223275e0, 0xc42207bc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4223275e0, 0xc42207bd48, 0xc4230c0e60, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4230c0e60, 0x4c, 0x0, 0x0, 0x0, 0xc4206f9801, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4230c0e60, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4230c0e60, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc424bd6ec0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x18, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1679 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e4190, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228ccc18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228ccc18, 0xc421c48000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228ccc00, 0xc421c48000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228ccc00, 0xc421c48000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef1f0, 0xc421c48000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4222cc7b0, 0x77aa1a0, 0xc4206ef1f0, 0x5, 0xc4206ef1f0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42251d880, 0x5d2d117, 0xc42251d9a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42251d880, 0xc421b41000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422980ae0, 0xc422b6c2e0, 0x5, 0x200, 0xc4230516e8, 0x6, 0xc42251da50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422980ae0, 0xc422b6c2e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc421ebe140, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422980ae0, 0xc422b6c2e0, 0x5, 0x200, 0x9391c20, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6c2c0, 0xc421ebe140, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6c2c0, 0xc421ebe140, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6c2c0, 0xc423051890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6c2c0, 0xc4218deb90, 0x4c, 0xc423051960, 0xc423051910, 0x4047467, 0x0, 0x5a02440, 0xc423051901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6c2c0, 0xc4218deb90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6c2c0, 0x6a55220, 0xc4200180f8, 0xc4218deb90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6c2c0, 0xc4218deb90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc421044770, 0xc423051c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc421044770, 0xc423051d48, 0xc4218deb90, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4218deb90, 0x4c, 0x0, 0x0, 0x0, 0xc4206f9801, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4218deb90, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc4218deb90, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc421ebe120, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x19, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1680 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e9868, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420f43098, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420f43098, 0xc421287400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420f43080, 0xc421287400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420f43080, 0xc421287400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc423142ef0, 0xc421287400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4225f8810, 0x77aa1a0, 0xc423142ef0, 0x5, 0xc423142ef0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42232f500, 0x5d2d117, 0xc42232f620, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42232f500, 0xc423892000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc420c713e0, 0xc420f98860, 0x5, 0x200, 0xc421a016e8, 0x6, 0xc42232f6d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc420c713e0, 0xc420f98860, 0x5, 0x200, 0x5, 0x40147f7, 0xc4223a1b40, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc420c713e0, 0xc420f98860, 0x5, 0x200, 0x73e69e0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420f98840, 0xc4223a1b40, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420f98840, 0xc4223a1b40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420f98840, 0xc421a01890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420f98840, 0xc42376b090, 0x4c, 0xc421a01960, 0xc421a01910, 0x4047467, 0x0, 0x5a02440, 0xc421a01901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420f98840, 0xc42376b090, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420f98840, 0x6a55220, 0xc4200180f8, 0xc42376b090, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420f98840, 0xc42376b090, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420269260, 0xc421a01c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420269260, 0xc421a01d48, 0xc42376b090, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42376b090, 0x4c, 0x0, 0x0, 0x0, 0xc420034701, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42376b090, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc42376b090, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4223a1b00, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x1a, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1681 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724d80, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a97598, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a97598, 0xc422916800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a97580, 0xc422916800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a97580, 0xc422916800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8370, 0xc422916800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420732d20, 0x77aa1a0, 0xc4204f8370, 0x5, 0xc4204f8370, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206a7500, 0x5d2d117, 0xc4206a7620, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206a7500, 0xc42291c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ac9c0, 0xc420cb2b20, 0x5, 0x200, 0xc421a056e8, 0x6, 0xc4206a76d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ac9c0, 0xc420cb2b20, 0x5, 0x200, 0x5, 0x40147f7, 0xc421c02c40, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ac9c0, 0xc420cb2b20, 0x5, 0x200, 0x73dbba0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420cb2b00, 0xc421c02c40, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420cb2b00, 0xc421c02c40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420cb2b00, 0xc421a05890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420cb2b00, 0xc422b17a90, 0x4c, 0xc421a05960, 0xc421a05910, 0x4047467, 0x0, 0x5a02440, 0xc421a05901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420cb2b00, 0xc422b17a90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420cb2b00, 0x6a55220, 0xc4200180f8, 0xc422b17a90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420cb2b00, 0xc422b17a90, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4223cf340, 0xc421a05c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4223cf340, 0xc421a05d48, 0xc422b17a90, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422b17a90, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422b17a90, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc422b17a90, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc421c02c00, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x1b, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1682 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x938e7d8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4222d8f98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4222d8f98, 0xc423633400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4222d8f80, 0xc423633400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4222d8f80, 0xc423633400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc421ac92f0, 0xc423633400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4237ffec0, 0x77aa1a0, 0xc421ac92f0, 0x5, 0xc421ac92f0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422e28e00, 0x5d2d117, 0xc422e28f20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422e28e00, 0xc421bf1000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422cd72c0, 0xc422b61620, 0x5, 0x200, 0xc422a2f6e8, 0x6, 0xc422e28fd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422cd72c0, 0xc422b61620, 0x5, 0x200, 0x5, 0x40147f7, 0xc421379e40, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422cd72c0, 0xc422b61620, 0x5, 0x200, 0x9371548, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b61600, 0xc421379e40, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b61600, 0xc421379e40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b61600, 0xc422a2f890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b61600, 0xc4232cdd10, 0x4c, 0xc422a2f960, 0xc422a2f910, 0x4047467, 0x0, 0x5a02440, 0xc422a2f901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b61600, 0xc4232cdd10, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b61600, 0x6a55220, 0xc4200180f8, 0xc4232cdd10, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b61600, 0xc4232cdd10, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4225a5730, 0xc422a2fc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4225a5730, 0xc422a2fd48, 0xc4232cdd10, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4232cdd10, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4232cdd10, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc4232cdd10, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc421379de0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x1c, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1683 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eb5e8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cd518, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cd518, 0xc421055800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cd500, 0xc421055800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cd500, 0xc421055800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e2c8, 0xc421055800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc421013f20, 0x77aa1a0, 0xc42000e2c8, 0x5, 0xc42000e2c8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206a6380, 0x5d2d117, 0xc4206a64a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206a6380, 0xc4206a9000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ac660, 0xc422b602e0, 0x5, 0x200, 0xc4234976e8, 0x6, 0xc4206a6550) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ac660, 0xc422b602e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc4215e38c0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ac660, 0xc422b602e0, 0x5, 0x200, 0x73ee840, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b602c0, 0xc4215e38c0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b602c0, 0xc4215e38c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b602c0, 0xc423497890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b602c0, 0xc421ab7220, 0x4c, 0xc423497960, 0xc423497910, 0x4047467, 0x0, 0x5a02440, 0xc423497901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b602c0, 0xc421ab7220, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b602c0, 0x6a55220, 0xc4200180f8, 0xc421ab7220, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b602c0, 0xc421ab7220, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc422528ee0, 0xc423497c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422528ee0, 0xc423497d48, 0xc421ab7220, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc421ab7220, 0x4c, 0x0, 0x0, 0x0, 0xc4202cce01, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc421ab7220, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc421ab7220, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4215e38a0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x1d, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1684 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e97a8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420462798, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420462798, 0xc421ee7000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420462780, 0xc421ee7000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420462780, 0xc421ee7000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8118, 0xc421ee7000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420162c00, 0x77aa1a0, 0xc4204f8118, 0x5, 0xc4204f8118, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42200b500, 0x5d2d117, 0xc42200b620, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42200b500, 0xc422591000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4223085a0, 0xc422b6dba0, 0x5, 0x200, 0xc4214016e8, 0x6, 0xc42200b6d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4223085a0, 0xc422b6dba0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422c0d0e0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4223085a0, 0xc422b6dba0, 0x5, 0x200, 0x77acca8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6db80, 0xc422c0d0e0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6db80, 0xc422c0d0e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6db80, 0xc421401890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6db80, 0xc4218df130, 0x4c, 0xc421401960, 0xc421401910, 0x4047467, 0x0, 0x5a02440, 0xc421401901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6db80, 0xc4218df130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6db80, 0x6a55220, 0xc4200180f8, 0xc4218df130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6db80, 0xc4218df130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4213a7340, 0xc421401c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4213a7340, 0xc421401d48, 0xc4218df130, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4218df130, 0x4c, 0x0, 0x0, 0x0, 0xc421945101, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4218df130, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4218df130, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422c0d0c0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x1e, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1685 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73ea3a8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cd418, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cd418, 0xc422af5000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cd400, 0xc422af5000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cd400, 0xc422af5000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f460, 0xc422af5000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4222cc8d0, 0x77aa1a0, 0xc42000f460, 0x5, 0xc42000f460, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421a0c700, 0x5d2d117, 0xc421a0c820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421a0c700, 0xc422723000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422980c00, 0xc422b6d8e0, 0x5, 0x200, 0xc42167d6e8, 0x6, 0xc421a0c8d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422980c00, 0xc422b6d8e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc42048eea0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422980c00, 0xc422b6d8e0, 0x5, 0x200, 0x7700558, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6d8c0, 0xc42048eea0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6d8c0, 0xc42048eea0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6d8c0, 0xc42167d890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6d8c0, 0xc42357cbe0, 0x4c, 0xc42167d960, 0xc42167d910, 0x4047467, 0x0, 0x5a02440, 0xc42167d901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6d8c0, 0xc42357cbe0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6d8c0, 0x6a55220, 0xc4200180f8, 0xc42357cbe0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6d8c0, 0xc42357cbe0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4204a21c0, 0xc42167dc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4204a21c0, 0xc42167dd48, 0xc42357cbe0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc42357cbe0, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc42357cbe0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc42357cbe0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc42048ed80, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x1f, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1686 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e9c28, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cc598, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cc598, 0xc42106d000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cc580, 0xc42106d000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cc580, 0xc42106d000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1f8, 0xc42106d000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420732240, 0x77aa1a0, 0xc4206ee1f8, 0x5, 0xc4206ee1f8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206a6e00, 0x5d2d117, 0xc4206a6f20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206a6e00, 0xc4211dc000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ac780, 0xc422c21360, 0x5, 0x200, 0xc4216796e8, 0x6, 0xc4206a6fd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ac780, 0xc422c21360, 0x5, 0x200, 0x5, 0x40147f7, 0xc42037cfa0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ac780, 0xc422c21360, 0x5, 0x200, 0x73c5bd0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422c21340, 0xc42037cfa0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422c21340, 0xc42037cfa0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422c21340, 0xc421679890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422c21340, 0xc422e6c0a0, 0x4c, 0xc421679960, 0xc421679910, 0x4047467, 0x0, 0x5a02440, 0xc421679901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422c21340, 0xc422e6c0a0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422c21340, 0x6a55220, 0xc4200180f8, 0xc422e6c0a0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422c21340, 0xc422e6c0a0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4211f0e70, 0xc421679c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4211f0e70, 0xc421679d48, 0xc422e6c0a0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422e6c0a0, 0x4c, 0x0, 0x0, 0x0, 0xc421945101, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422e6c0a0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc422e6c0a0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc42037cf80, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x20, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1687 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73ea2e8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cd618, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cd618, 0xc422d7f000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cd600, 0xc422d7f000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cd600, 0xc422d7f000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f4e0, 0xc422d7f000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4222c96b0, 0x77aa1a0, 0xc42000f4e0, 0x5, 0xc42000f4e0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420edce00, 0x5d2d117, 0xc420edcf20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420edce00, 0xc4223e1000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228f3140, 0xc422c20020, 0x5, 0x200, 0xc4217a36e8, 0x6, 0xc420edcfd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228f3140, 0xc422c20020, 0x5, 0x200, 0x5, 0x40147f7, 0xc420397a40, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228f3140, 0xc422c20020, 0x5, 0x200, 0x73c6368, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422c20000, 0xc420397a40, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422c20000, 0xc420397a40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422c20000, 0xc4217a3890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422c20000, 0xc4226c66e0, 0x4c, 0xc4217a3960, 0xc4217a3910, 0x4047467, 0x0, 0x5a02440, 0xc4217a3901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422c20000, 0xc4226c66e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422c20000, 0x6a55220, 0xc4200180f8, 0xc4226c66e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422c20000, 0xc4226c66e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420cea540, 0xc4217a3c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc420cea540, 0xc4217a3d48, 0xc4226c66e0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4226c66e0, 0x4c, 0x0, 0x0, 0x0, 0xc420064301, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4226c66e0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4226c66e0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc420397a00, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x21, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1688 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724900, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421accc98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421accc98, 0xc422ffe000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421accc80, 0xc422ffe000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421accc80, 0xc422ffe000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc423142d50, 0xc422ffe000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc422e94c60, 0x77aa1a0, 0xc423142d50, 0x5, 0xc423142d50, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422dd2e00, 0x5d2d117, 0xc422dd2f20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422dd2e00, 0xc423180000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc423409980, 0xc422c21ba0, 0x5, 0x200, 0xc4248876e8, 0x6, 0xc422dd2fd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc423409980, 0xc422c21ba0, 0x5, 0x200, 0x5, 0x40147f7, 0xc421379c00, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc423409980, 0xc422c21ba0, 0x5, 0x200, 0x9371548, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422c21b80, 0xc421379c00, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422c21b80, 0xc421379c00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422c21b80, 0xc424887890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422c21b80, 0xc422e6c870, 0x4c, 0xc424887960, 0xc424887910, 0x4047467, 0x0, 0x5a02440, 0xc424887901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422c21b80, 0xc422e6c870, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422c21b80, 0x6a55220, 0xc4200180f8, 0xc422e6c870, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422c21b80, 0xc422e6c870, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420343c00, 0xc424887c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc420343c00, 0xc424887d48, 0xc422e6c870, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422e6c870, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422e6c870, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc422e6c870, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc421379be0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x22, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1689 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eb0a8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420463d18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420463d18, 0xc421b34c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420463d00, 0xc421b34c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420463d00, 0xc421b34c00, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0338, 0xc421b34c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4204d82d0, 0x77aa1a0, 0xc4205b0338, 0x5, 0xc4205b0338, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224f7c00, 0x5d2d117, 0xc4224f7d20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224f7c00, 0xc421b39000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422981f80, 0xc4210fa860, 0x5, 0x200, 0xc4225ad6e8, 0x6, 0xc4224f7dd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422981f80, 0xc4210fa860, 0x5, 0x200, 0x5, 0x40147f7, 0xc420358200, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422981f80, 0xc4210fa860, 0x5, 0x200, 0x73c5120, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc4210fa840, 0xc420358200, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc4210fa840, 0xc420358200, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc4210fa840, 0xc4225ad890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc4210fa840, 0xc42226a460, 0x4c, 0xc4225ad960, 0xc4225ad910, 0x4047467, 0x0, 0x5a02440, 0xc4225ad901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc4210fa840, 0xc42226a460, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc4210fa840, 0x6a55220, 0xc4200180f8, 0xc42226a460, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc4210fa840, 0xc42226a460, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420343d50, 0xc4225adc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420343d50, 0xc4225add48, 0xc42226a460, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42226a460, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42226a460, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc42226a460, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4203581e0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x23, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1690 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e9ce8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cc398, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cc398, 0xc422668000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cc380, 0xc422668000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cc380, 0xc422668000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee2c0, 0xc422668000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420162cf0, 0x77aa1a0, 0xc4206ee2c0, 0x5, 0xc4206ee2c0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfa000, 0x5d2d117, 0xc421bfa120, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfa000, 0xc421c53000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308660, 0xc422c20de0, 0x5, 0x200, 0xc421c1d6e8, 0x6, 0xc421bfa1d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308660, 0xc422c20de0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422b06fa0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308660, 0xc422c20de0, 0x5, 0x200, 0x9392210, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422c20dc0, 0xc422b06fa0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422c20dc0, 0xc422b06fa0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422c20dc0, 0xc421c1d890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422c20dc0, 0xc4228bc370, 0x4c, 0xc421c1d960, 0xc421c1d910, 0x4047467, 0x0, 0x5a02440, 0xc421c1d901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422c20dc0, 0xc4228bc370, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422c20dc0, 0x6a55220, 0xc4200180f8, 0xc4228bc370, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422c20dc0, 0xc4228bc370, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420f6b9d0, 0xc421c1dc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc420f6b9d0, 0xc421c1dd48, 0xc4228bc370, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4228bc370, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4228bc370, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4228bc370, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422b06f80, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x24, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1691 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eb228, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420463498, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420463498, 0xc4218f7800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420463480, 0xc4218f7800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420463480, 0xc4218f7800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b03f0, 0xc4218f7800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f6c4b0, 0x77aa1a0, 0xc4205b03f0, 0x5, 0xc4205b03f0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42204d880, 0x5d2d117, 0xc42204d9a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42204d880, 0xc4220cb000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc421fb77a0, 0xc422311ba0, 0x5, 0x200, 0xc4226036e8, 0x6, 0xc42204da50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc421fb77a0, 0xc422311ba0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422988ba0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc421fb77a0, 0xc422311ba0, 0x5, 0x200, 0x77bc5c8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422311b80, 0xc422988ba0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422311b80, 0xc422988ba0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422311b80, 0xc422603890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422311b80, 0xc422b17d10, 0x4c, 0xc422603960, 0xc422603910, 0x4047467, 0x0, 0x5a02440, 0xc422603901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422311b80, 0xc422b17d10, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422311b80, 0x6a55220, 0xc4200180f8, 0xc422b17d10, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422311b80, 0xc422b17d10, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc421044000, 0xc422603c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc421044000, 0xc422603d48, 0xc422b17d10, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422b17d10, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422b17d10, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc422b17d10, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422988b80, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x25, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1692 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eb2e8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420463298, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420463298, 0xc422624000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420463280, 0xc422624000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420463280, 0xc422624000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0418, 0xc422624000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f6da10, 0x77aa1a0, 0xc4205b0418, 0x5, 0xc4205b0418, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c59c00, 0x5d2d117, 0xc421c59d20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c59c00, 0xc421f03000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422995020, 0xc422311360, 0x5, 0x200, 0xc4226076e8, 0x6, 0xc421c59dd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422995020, 0xc422311360, 0x5, 0x200, 0x5, 0x40147f7, 0xc4224335c0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422995020, 0xc422311360, 0x5, 0x200, 0x77b4288, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422311340, 0xc4224335c0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422311340, 0xc4224335c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422311340, 0xc422607890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422311340, 0xc4229ed810, 0x4c, 0xc422607960, 0xc422607910, 0x4047467, 0x0, 0x5a02440, 0xc422607901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422311340, 0xc4229ed810, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422311340, 0x6a55220, 0xc4200180f8, 0xc4229ed810, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422311340, 0xc4229ed810, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4213a69a0, 0xc422607c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4213a69a0, 0xc422607d48, 0xc4229ed810, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4229ed810, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4229ed810, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4229ed810, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4224335a0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x26, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1693 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e4850, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208bb018, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208bb018, 0xc421c48800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208bb000, 0xc421c48800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208bb000, 0xc421c48800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8ca0, 0xc421c48800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420d24060, 0x77aa1a0, 0xc4204f8ca0, 0x5, 0xc4204f8ca0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420edd180, 0x5d2d117, 0xc420edd2a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420edd180, 0xc421b10000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308840, 0xc422311620, 0x5, 0x200, 0xc4215b96e8, 0x6, 0xc420edd350) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308840, 0xc422311620, 0x5, 0x200, 0x5, 0x40147f7, 0xc422860da0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308840, 0xc422311620, 0x5, 0x200, 0x73ef5e8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422311600, 0xc422860da0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422311600, 0xc422860da0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422311600, 0xc4215b9890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422311600, 0xc422e2b3b0, 0x4c, 0xc4215b9960, 0xc4215b9910, 0x4047467, 0x0, 0x5a02440, 0xc4215b9901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422311600, 0xc422e2b3b0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422311600, 0x6a55220, 0xc4200180f8, 0xc422e2b3b0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422311600, 0xc422e2b3b0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc421036540, 0xc4215b9c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc421036540, 0xc4215b9d48, 0xc422e2b3b0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e2b3b0, 0x4c, 0x0, 0x0, 0x0, 0xc4206f9801, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e2b3b0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc422e2b3b0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422860d80, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x27, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1694 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7725080, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96d98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96d98, 0xc4228dfc00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96d80, 0xc4228dfc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96d80, 0xc4228dfc00, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b04d8, 0xc4228dfc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4204e19e0, 0x77aa1a0, 0xc4205b04d8, 0x5, 0xc4205b04d8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224f7180, 0x5d2d117, 0xc4224f72a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224f7180, 0xc422af7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422981ec0, 0xc42134d360, 0x5, 0x200, 0xc4215bd6e8, 0x6, 0xc4224f7350) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422981ec0, 0xc42134d360, 0x5, 0x200, 0x5, 0x40147f7, 0xc422c0c560, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422981ec0, 0xc42134d360, 0x5, 0x200, 0x77acca8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc42134d340, 0xc422c0c560, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc42134d340, 0xc422c0c560, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc42134d340, 0xc4215bd890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc42134d340, 0xc422df46e0, 0x4c, 0xc4215bd960, 0xc4215bd910, 0x4047467, 0x0, 0x5a02440, 0xc4215bd901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc42134d340, 0xc422df46e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc42134d340, 0x6a55220, 0xc4200180f8, 0xc422df46e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc42134d340, 0xc422df46e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420b8a7e0, 0xc4215bdc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc420b8a7e0, 0xc4215bdd48, 0xc422df46e0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422df46e0, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422df46e0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc422df46e0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422c0c540, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x28, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1695 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724a80, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a97d98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a97d98, 0xc421ffe000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a97d80, 0xc421ffe000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a97d80, 0xc421ffe000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f82d0, 0xc421ffe000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4204d8a80, 0x77aa1a0, 0xc4204f82d0, 0x5, 0xc4204f82d0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42204c380, 0x5d2d117, 0xc42204c4a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42204c380, 0xc421b57000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc421fb62a0, 0xc4211382e0, 0x5, 0x200, 0xc4222fb6e8, 0x6, 0xc42204c550) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc421fb62a0, 0xc4211382e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422e7b360, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc421fb62a0, 0xc4211382e0, 0x5, 0x200, 0x77bf948, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc4211382c0, 0xc422e7b360, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc4211382c0, 0xc422e7b360, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc4211382c0, 0xc4222fb890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc4211382c0, 0xc42357c280, 0x4c, 0xc4222fb960, 0xc4222fb910, 0x4047467, 0x0, 0x5a02440, 0xc4222fb901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc4211382c0, 0xc42357c280, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc4211382c0, 0x6a55220, 0xc4200180f8, 0xc42357c280, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc4211382c0, 0xc42357c280, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc422528e70, 0xc4222fbc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422528e70, 0xc4222fbd48, 0xc42357c280, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42357c280, 0x4c, 0x0, 0x0, 0x0, 0xc420064301, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc42357c280, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc42357c280, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422e7b300, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x29, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1696 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e9628, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cd298, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cd298, 0xc4228dec00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cd280, 0xc4228dec00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cd280, 0xc4228dec00, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e340, 0xc4228dec00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4204e12f0, 0x77aa1a0, 0xc42000e340, 0x5, 0xc42000e340, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224f6380, 0x5d2d117, 0xc4224f64a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224f6380, 0xc42259b000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422981e00, 0xc420f998e0, 0x5, 0x200, 0xc422d196e8, 0x6, 0xc4224f6550) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422981e00, 0xc420f998e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc420365b40, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422981e00, 0xc420f998e0, 0x5, 0x200, 0x73c54b0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420f998c0, 0xc420365b40, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420f998c0, 0xc420365b40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420f998c0, 0xc422d19890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420f998c0, 0xc4230c0f00, 0x4c, 0xc422d19960, 0xc422d19910, 0x4047467, 0x0, 0x5a02440, 0xc422d19901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420f998c0, 0xc4230c0f00, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420f998c0, 0x6a55220, 0xc4200180f8, 0xc4230c0f00, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420f998c0, 0xc4230c0f00, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc421281490, 0xc422d19c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc421281490, 0xc422d19d48, 0xc4230c0f00, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4230c0f00, 0x4c, 0x0, 0x0, 0x0, 0xc422dd2001, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4230c0f00, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4230c0f00, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4203659c0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x2a, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1697 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e4610, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c32c98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c32c98, 0xc421fd1800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c32c80, 0xc421fd1800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c32c80, 0xc421fd1800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f438, 0xc421fd1800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4222cc690, 0x77aa1a0, 0xc42000f438, 0x5, 0xc42000f438, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42251d500, 0x5d2d117, 0xc42251d620, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42251d500, 0xc421b11000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422980a80, 0xc422abd360, 0x5, 0x200, 0xc424ce36e8, 0x6, 0xc42251d6d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422980a80, 0xc422abd360, 0x5, 0x200, 0x5, 0x40147f7, 0xc421fa9700, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422980a80, 0xc422abd360, 0x5, 0x200, 0x7709230, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422abd340, 0xc421fa9700, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422abd340, 0xc421fa9700, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422abd340, 0xc424ce3890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422abd340, 0xc42551cdc0, 0x4c, 0xc424ce3960, 0xc424ce3910, 0x4047467, 0x0, 0x5a02440, 0xc424ce3901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422abd340, 0xc42551cdc0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422abd340, 0x6a55220, 0xc4200180f8, 0xc42551cdc0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422abd340, 0xc42551cdc0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420e75420, 0xc424ce3c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc420e75420, 0xc424ce3d48, 0xc42551cdc0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc42551cdc0, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc42551cdc0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc42551cdc0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc421fa96e0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x2b, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1698 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x7724f00, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a97198, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a97198, 0xc4219d8800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a97180, 0xc4219d8800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a97180, 0xc4219d8800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b0480, 0xc4219d8800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f6cb70, 0x77aa1a0, 0xc4205b0480, 0x5, 0xc4205b0480, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42232ea80, 0x5d2d117, 0xc42232eba0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42232ea80, 0xc421b51000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ad320, 0xc420cb2020, 0x5, 0x200, 0xc421f0b6e8, 0x6, 0xc42232ec50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ad320, 0xc420cb2020, 0x5, 0x200, 0x5, 0x40147f7, 0xc421ebfae0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ad320, 0xc420cb2020, 0x5, 0x200, 0x9391c20, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420cb2000, 0xc421ebfae0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420cb2000, 0xc421ebfae0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420cb2000, 0xc421f0b890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420cb2000, 0xc4228bd310, 0x4c, 0xc421f0b960, 0xc421f0b910, 0x4047467, 0x0, 0x5a02440, 0xc421f0b901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420cb2000, 0xc4228bd310, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420cb2000, 0x6a55220, 0xc4200180f8, 0xc4228bd310, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420cb2000, 0xc4228bd310, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420f70fc0, 0xc421f0bc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420f70fc0, 0xc421f0bd48, 0xc4228bd310, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4228bd310, 0x4c, 0x0, 0x0, 0x0, 0xc4206f9801, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4228bd310, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4228bd310, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc421ebfac0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x2c, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1699 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eab68, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421970a18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421970a18, 0xc421695000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421970a00, 0xc421695000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421970a00, 0xc421695000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8268, 0xc421695000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420b7e7e0, 0x77aa1a0, 0xc4204f8268, 0x5, 0xc4204f8268, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfb880, 0x5d2d117, 0xc421bfb9a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfb880, 0xc422ae8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308e40, 0xc420012b20, 0x5, 0x200, 0xc421f0f6e8, 0x6, 0xc421bfba50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308e40, 0xc420012b20, 0x5, 0x200, 0x5, 0x40147f7, 0xc422e8ea20, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308e40, 0xc420012b20, 0x5, 0x200, 0x77bff38, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc420012b00, 0xc422e8ea20, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc420012b00, 0xc422e8ea20, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc420012b00, 0xc421f0f890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc420012b00, 0xc422e12f50, 0x4c, 0xc421f0f960, 0xc421f0f910, 0x4047467, 0x0, 0x5a02440, 0xc421f0f901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc420012b00, 0xc422e12f50, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc420012b00, 0x6a55220, 0xc4200180f8, 0xc422e12f50, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc420012b00, 0xc422e12f50, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc422528a80, 0xc421f0fc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422528a80, 0xc421f0fd48, 0xc422e12f50, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e12f50, 0x4c, 0x0, 0x0, 0x0, 0xc421945101, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e12f50, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc422e12f50, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422e8ea00, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x2d, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1700 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e4250, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c33398, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c33398, 0xc422af4800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c33380, 0xc422af4800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c33380, 0xc422af4800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef1c8, 0xc422af4800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4222cc870, 0x77aa1a0, 0xc4206ef1c8, 0x5, 0xc4206ef1c8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421a0c000, 0x5d2d117, 0xc421a0c120, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421a0c000, 0xc422722000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422980ba0, 0xc422b605a0, 0x5, 0x200, 0xc4218d76e8, 0x6, 0xc421a0c1d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422980ba0, 0xc422b605a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc421209100, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422980ba0, 0xc422b605a0, 0x5, 0x200, 0x7712bf0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b60580, 0xc421209100, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b60580, 0xc421209100, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b60580, 0xc4218d7890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b60580, 0xc423e93400, 0x4c, 0xc4218d7960, 0xc4218d7910, 0x4047467, 0x0, 0x5a02440, 0xc4218d7901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b60580, 0xc423e93400, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b60580, 0x6a55220, 0xc4200180f8, 0xc423e93400, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b60580, 0xc423e93400, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4200df030, 0xc4218d7c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4200df030, 0xc4218d7d48, 0xc423e93400, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc423e93400, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc423e93400, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc423e93400, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4212090c0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x2e, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1701 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e4490, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c32e98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c32e98, 0xc422054800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c32e80, 0xc422054800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c32e80, 0xc422054800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f410, 0xc422054800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2f8f0, 0x77aa1a0, 0xc42000f410, 0x5, 0xc42000f410, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b17500, 0x5d2d117, 0xc420b17620, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b17500, 0xc4224c2000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308060, 0xc422abd8e0, 0x5, 0x200, 0xc4226e16e8, 0x6, 0xc420b176d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308060, 0xc422abd8e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc422e246e0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308060, 0xc422abd8e0, 0x5, 0x200, 0x9396e10, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422abd8c0, 0xc422e246e0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422abd8c0, 0xc422e246e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422abd8c0, 0xc4226e1890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422abd8c0, 0xc4229a6c30, 0x4c, 0xc4226e1960, 0xc4226e1910, 0x4047467, 0x0, 0x5a02440, 0xc4226e1901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422abd8c0, 0xc4229a6c30, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422abd8c0, 0x6a55220, 0xc4200180f8, 0xc4229a6c30, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422abd8c0, 0xc4229a6c30, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420f707e0, 0xc4226e1c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420f707e0, 0xc4226e1d48, 0xc4229a6c30, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4229a6c30, 0x4c, 0x0, 0x0, 0x0, 0xc4202cce01, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4229a6c30, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4229a6c30, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422e246c0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x2f, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1702 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e3e90, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c33598, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c33598, 0xc4228df400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c33580, 0xc4228df400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c33580, 0xc4228df400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000f490, 0xc4228df400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4204e15c0, 0x77aa1a0, 0xc42000f490, 0x5, 0xc42000f490, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224f6700, 0x5d2d117, 0xc4224f6820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224f6700, 0xc4229c4000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422981e60, 0xc422b60b20, 0x5, 0x200, 0xc422df16e8, 0x6, 0xc4224f68d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422981e60, 0xc422b60b20, 0x5, 0x200, 0x5, 0x40147f7, 0xc422d4c400, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422981e60, 0xc422b60b20, 0x5, 0x200, 0x73e0490, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b60b00, 0xc422d4c400, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b60b00, 0xc422d4c400, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b60b00, 0xc422df1890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b60b00, 0xc423689130, 0x4c, 0xc422df1960, 0xc422df1910, 0x4047467, 0x0, 0x5a02440, 0xc422df1901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b60b00, 0xc423689130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b60b00, 0x6a55220, 0xc4200180f8, 0xc423689130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b60b00, 0xc423689130, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420f6be30, 0xc422df1c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc420f6be30, 0xc422df1d48, 0xc423689130, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc423689130, 0x4c, 0x0, 0x0, 0x0, 0xc420034701, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc423689130, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc423689130, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422d4c3e0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x30, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1703 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e9b68, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cc798, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cc798, 0xc4224db000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cc780, 0xc4224db000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cc780, 0xc4224db000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee168, 0xc4224db000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2fd70, 0x77aa1a0, 0xc4206ee168, 0x5, 0xc4206ee168, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422527500, 0x5d2d117, 0xc422527620, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422527500, 0xc42194e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308480, 0xc422c218e0, 0x5, 0x200, 0xc4226dd6e8, 0x6, 0xc4225276d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308480, 0xc422c218e0, 0x5, 0x200, 0x5, 0x40147f7, 0xc4203e5540, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308480, 0xc422c218e0, 0x5, 0x200, 0x73c7a90, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422c218c0, 0xc4203e5540, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422c218c0, 0xc4203e5540, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422c218c0, 0xc4226dd890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422c218c0, 0xc4226c7cc0, 0x4c, 0xc4226dd960, 0xc4226dd910, 0x4047467, 0x0, 0x5a02440, 0xc4226dd901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422c218c0, 0xc4226c7cc0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422c218c0, 0x6a55220, 0xc4200180f8, 0xc4226c7cc0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422c218c0, 0xc4226c7cc0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4212460e0, 0xc4226ddc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4212460e0, 0xc4226ddd48, 0xc4226c7cc0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4226c7cc0, 0x4c, 0x0, 0x0, 0x0, 0xc4206f9801, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4226c7cc0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc4226c7cc0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4203e5500, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x31, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1704 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e46d0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c32a18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c32a18, 0xc421055000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c32a00, 0xc421055000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c32a00, 0xc421055000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef198, 0xc421055000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc421013e90, 0x77aa1a0, 0xc4206ef198, 0x5, 0xc4206ef198, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4206a6000, 0x5d2d117, 0xc4206a6120, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4206a6000, 0xc4206a8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228ac600, 0xc422abcde0, 0x5, 0x200, 0xc421dfb6e8, 0x6, 0xc4206a61d0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228ac600, 0xc422abcde0, 0x5, 0x200, 0x5, 0x40147f7, 0xc421b1cba0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228ac600, 0xc422abcde0, 0x5, 0x200, 0x938b500, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422abcdc0, 0xc421b1cba0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422abcdc0, 0xc421b1cba0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422abcdc0, 0xc421dfb890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422abcdc0, 0xc4220814a0, 0x4c, 0xc421dfb960, 0xc421dfb910, 0x4047467, 0x0, 0x5a02440, 0xc421dfb901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422abcdc0, 0xc4220814a0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422abcdc0, 0x6a55220, 0xc4200180f8, 0xc4220814a0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422abcdc0, 0xc4220814a0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc420e75730, 0xc421dfbc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc420e75730, 0xc421dfbd48, 0xc4220814a0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4220814a0, 0x4c, 0x0, 0x0, 0x0, 0xc420466001, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4220814a0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc4220814a0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc421b1cb80, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x32, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1705 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x938f498, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420f42398, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420f42398, 0xc422749400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420f42380, 0xc422749400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420f42380, 0xc422749400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e680, 0xc422749400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420bd43f0, 0x77aa1a0, 0xc42000e680, 0x5, 0xc42000e680, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfbc00, 0x5d2d117, 0xc421bfbd20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfbc00, 0xc422ae9000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4223093e0, 0xc421138de0, 0x5, 0x200, 0xc421df76e8, 0x6, 0xc421bfbdd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4223093e0, 0xc421138de0, 0x5, 0x200, 0x5, 0x40147f7, 0xc4215e3340, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4223093e0, 0xc421138de0, 0x5, 0x200, 0x73ee840, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc421138dc0, 0xc4215e3340, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc421138dc0, 0xc4215e3340, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc421138dc0, 0xc421df7890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc421138dc0, 0xc4226c72c0, 0x4c, 0xc421df7960, 0xc421df7910, 0x4047467, 0x0, 0x5a02440, 0xc421df7901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc421138dc0, 0xc4226c72c0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc421138dc0, 0x6a55220, 0xc4200180f8, 0xc4226c72c0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc421138dc0, 0xc4226c72c0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4225289a0, 0xc421df7c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4225289a0, 0xc421df7d48, 0xc4226c72c0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4226c72c0, 0x4c, 0x0, 0x0, 0x0, 0xc420034701, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4226c72c0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc4226c72c0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4215e3300, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x33, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1706 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eb528, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cdc18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cdc18, 0xc42228d800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cdc00, 0xc42228d800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cdc00, 0xc42228d800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e230, 0xc42228d800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2fc50, 0x77aa1a0, 0xc42000e230, 0x5, 0xc42000e230, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422526a80, 0x5d2d117, 0xc422526ba0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422526a80, 0xc421f69000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308360, 0xc422b60de0, 0x5, 0x200, 0xc4233836e8, 0x6, 0xc422526c50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308360, 0xc422b60de0, 0x5, 0x200, 0x5, 0x40147f7, 0xc42129e8e0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308360, 0xc422b60de0, 0x5, 0x200, 0x93705d8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b60dc0, 0xc42129e8e0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b60dc0, 0xc42129e8e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b60dc0, 0xc423383890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b60dc0, 0xc42376b400, 0x4c, 0xc423383960, 0xc423383910, 0x4047467, 0x0, 0x5a02440, 0xc423383901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b60dc0, 0xc42376b400, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b60dc0, 0x6a55220, 0xc4200180f8, 0xc42376b400, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b60dc0, 0xc42376b400, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc42193a460, 0xc423383c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc42193a460, 0xc423383d48, 0xc42376b400, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc42376b400, 0x4c, 0x0, 0x0, 0x0, 0xc421945101, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc42376b400, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc42376b400, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc42129e8c0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x34, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1707 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73ea228, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cd818, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cd818, 0xc421b0f000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cd800, 0xc421b0f000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cd800, 0xc421b0f000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef280, 0xc421b0f000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2f6b0, 0x77aa1a0, 0xc4206ef280, 0x5, 0xc4206ef280, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b16a80, 0x5d2d117, 0xc420b16ba0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b16a80, 0xc422941000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422243f20, 0xc422c205a0, 0x5, 0x200, 0xc42337f6e8, 0x6, 0xc420b16c50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422243f20, 0xc422c205a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc4223a19c0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422243f20, 0xc422c205a0, 0x5, 0x200, 0x73e69e0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422c20580, 0xc4223a19c0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422c20580, 0xc4223a19c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422c20580, 0xc42337f890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422c20580, 0xc422293a40, 0x4c, 0xc42337f960, 0xc42337f910, 0x4047467, 0x0, 0x5a02440, 0xc42337f901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422c20580, 0xc422293a40, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422c20580, 0x6a55220, 0xc4200180f8, 0xc422293a40, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422c20580, 0xc422293a40, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc421108e70, 0xc42337fc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc421108e70, 0xc42337fd48, 0xc422293a40, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422293a40, 0x4c, 0x0, 0x0, 0x0, 0xc421945101, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422293a40, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc422293a40, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4223a1900, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x35, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1708 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e4310, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c33098, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c33098, 0xc422d7e400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c33080, 0xc422d7e400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c33080, 0xc422d7e400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8d98, 0xc422d7e400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4222cdb00, 0x77aa1a0, 0xc4204f8d98, 0x5, 0xc4204f8d98, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c58e00, 0x5d2d117, 0xc421c58f20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c58e00, 0xc422d78000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422970fc0, 0xc422b60020, 0x5, 0x200, 0xc4216096e8, 0x6, 0xc421c58fd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422970fc0, 0xc422b60020, 0x5, 0x200, 0x5, 0x40147f7, 0xc42230b6a0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422970fc0, 0xc422b60020, 0x5, 0x200, 0x77ae3d0, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b60000, 0xc42230b6a0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b60000, 0xc42230b6a0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b60000, 0xc421609890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b60000, 0xc421f23220, 0x4c, 0xc421609960, 0xc421609910, 0x4047467, 0x0, 0x5a02440, 0xc421609901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b60000, 0xc421f23220, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b60000, 0x6a55220, 0xc4200180f8, 0xc421f23220, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b60000, 0xc421f23220, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4200df420, 0xc421609c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4200df420, 0xc421609d48, 0xc421f23220, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc421f23220, 0x4c, 0x0, 0x0, 0x0, 0xc421945101, 0x735b458, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc421f23220, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc421f23220, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc42230b660, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x36, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1709 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e96e8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4231fb098, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4231fb098, 0xc423ee6000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4231fb080, 0xc423ee6000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4231fb080, 0xc423ee6000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206efaf0, 0xc423ee6000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc42124e870, 0x77aa1a0, 0xc4206efaf0, 0x5, 0xc4206efaf0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422105880, 0x5d2d117, 0xc4221059a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422105880, 0xc421eeb000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422970e40, 0xc422c210a0, 0x5, 0x200, 0xc42428d6e8, 0x6, 0xc422105a50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422970e40, 0xc422c210a0, 0x5, 0x200, 0x5, 0x40147f7, 0xc42297a160, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422970e40, 0xc422c210a0, 0x5, 0x200, 0x771aa68, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422c21080, 0xc42297a160, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422c21080, 0xc42297a160, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422c21080, 0xc42428d890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422c21080, 0xc422e13630, 0x4c, 0xc42428d960, 0xc42428d910, 0x4047467, 0x0, 0x5a02440, 0xc42428d901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422c21080, 0xc422e13630, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422c21080, 0x6a55220, 0xc4200180f8, 0xc422e13630, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422c21080, 0xc422e13630, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc423bc19d0, 0xc42428dc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc423bc19d0, 0xc42428dd48, 0xc422e13630, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422e13630, 0x4c, 0x0, 0x0, 0x0, 0xc421945101, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc422e13630, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc422e13630, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc42297a140, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x37, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1710 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73ea528, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cd218, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cd218, 0xc422054000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cd200, 0xc422054000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cd200, 0xc422054000, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8de8, 0xc422054000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2f890, 0x77aa1a0, 0xc4204f8de8, 0x5, 0xc4204f8de8, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b17180, 0x5d2d117, 0xc420b172a0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b17180, 0xc422901000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422308000, 0xc422b6d360, 0x5, 0x200, 0xc4215fd6e8, 0x6, 0xc420b17350) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422308000, 0xc422b6d360, 0x5, 0x200, 0x5, 0x40147f7, 0xc422b07940, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422308000, 0xc422b6d360, 0x5, 0x200, 0x9392210, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6d340, 0xc422b07940, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6d340, 0xc422b07940, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6d340, 0xc4215fd890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6d340, 0xc423e92ff0, 0x4c, 0xc4215fd960, 0xc4215fd910, 0x4047467, 0x0, 0x5a02440, 0xc4215fd901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6d340, 0xc423e92ff0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6d340, 0x6a55220, 0xc4200180f8, 0xc423e92ff0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6d340, 0xc423e92ff0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4213a7b20, 0xc4215fdc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4213a7b20, 0xc4215fdd48, 0xc423e92ff0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc423e92ff0, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc423e92ff0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc423e92ff0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422b078e0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x38, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1711 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eb3a8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc423640998, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc423640998, 0xc424697400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc423640980, 0xc424697400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc423640980, 0xc424697400, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc421ac8b70, 0xc424697400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc422ed27b0, 0x77aa1a0, 0xc421ac8b70, 0x5, 0xc421ac8b70, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422dd2a80, 0x5d2d117, 0xc422dd2ba0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422dd2a80, 0xc42342c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc421239440, 0xc422b6cb20, 0x5, 0x200, 0xc42488b6e8, 0x6, 0xc422dd2c50) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc421239440, 0xc422b6cb20, 0x5, 0x200, 0x5, 0x40147f7, 0xc4227a9f80, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc421239440, 0xc422b6cb20, 0x5, 0x200, 0x9394188, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc422b6cb00, 0xc4227a9f80, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc422b6cb00, 0xc4227a9f80, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc422b6cb00, 0xc42488b890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc422b6cb00, 0xc422e2a960, 0x4c, 0xc42488b960, 0xc42488b910, 0x4047467, 0x0, 0x5a02440, 0xc42488b901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc422b6cb00, 0xc422e2a960, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc422b6cb00, 0x6a55220, 0xc4200180f8, 0xc422e2a960, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc422b6cb00, 0xc422e2a960, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4223cf1f0, 0xc42488bc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc4223cf1f0, 0xc42488bd48, 0xc422e2a960, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e2a960, 0x4c, 0x0, 0x0, 0x0, 0xc421945801, 0x735ad90, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420e09b80, 0x6a55220, 0xc4200180f8, 0xc422e2a960, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420e09b80, 0xc422e2a960, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4227a9f60, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x39, 0x3e8, 0xc420e09b80, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1712 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73eaaa8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421970e18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421970e18, 0xc422055800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421970e00, 0xc422055800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421970e00, 0xc422055800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8208, 0xc422055800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2faa0, 0x77aa1a0, 0xc4204f8208, 0x5, 0xc4204f8208, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b17c00, 0x5d2d117, 0xc420b17d20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b17c00, 0xc4224c7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4223081e0, 0xc4204c6860, 0x5, 0x200, 0xc4220316e8, 0x6, 0xc420b17dd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4223081e0, 0xc4204c6860, 0x5, 0x200, 0x5, 0x40147f7, 0xc422e25be0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4223081e0, 0xc4204c6860, 0x5, 0x200, 0x9396e10, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc4204c6840, 0xc422e25be0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc4204c6840, 0xc422e25be0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc4204c6840, 0xc422031890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc4204c6840, 0xc4239621e0, 0x4c, 0xc422031960, 0xc422031910, 0x4047467, 0x0, 0x5a02440, 0xc422031901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc4204c6840, 0xc4239621e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc4204c6840, 0x6a55220, 0xc4200180f8, 0xc4239621e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc4204c6840, 0xc4239621e0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4213a6930, 0xc422031c68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4213a6930, 0xc422031d48, 0xc4239621e0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4239621e0, 0x4c, 0x0, 0x0, 0x0, 0xc420034701, 0x735a000, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc420f44aa0, 0x6a55220, 0xc4200180f8, 0xc4239621e0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc420f44aa0, 0xc4239621e0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc422e25b80, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x3a, 0x3e8, 0xc420f44aa0, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1713 [IO wait, 1 minutes]: internal/poll.runtime_pollWait(0x73e43d0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4228cca18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4228cca18, 0xc421b0f800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4228cca00, 0xc421b0f800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4228cca00, 0xc421b0f800, 0x400, 0x400, 0x35, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8dc0, 0xc421b0f800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc420f2f710, 0x77aa1a0, 0xc4204f8dc0, 0x5, 0xc4204f8dc0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b16e00, 0x5d2d117, 0xc420b16f20, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b16e00, 0xc422900000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc422243f80, 0xc4204c7ba0, 0x5, 0x200, 0xc42202d6e8, 0x6, 0xc420b16fd0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc422243f80, 0xc4204c7ba0, 0x5, 0x200, 0x5, 0x40147f7, 0xc4211d96c0, 0x20) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc422243f80, 0xc4204c7ba0, 0x5, 0x200, 0x73d4cf8, 0x0, 0x0) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recvMessage(0xc4204c7b80, 0xc4211d96c0, 0x597e760, 0x4eddf01, 0x6cf0d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:947 +0x14e github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1Buf(0xc4204c7b80, 0xc4211d96c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:997 +0x39 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).recv1(0xc4204c7b80, 0xc42202d890, 0x4d) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:1018 +0x85 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).simpleExec(0xc4204c7b80, 0xc422292ff0, 0x4c, 0xc42202d960, 0xc42202d910, 0x4047467, 0x0, 0x5a02440, 0xc42202d901) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:619 +0x1cc github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).Exec(0xc4204c7b80, 0xc422292ff0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn.go:875 +0x272 github.com/cockroachdb/cockroach/vendor/github.com/lib/pq.(*conn).ExecContext(0xc4204c7b80, 0x6a55220, 0xc4200180f8, 0xc422292ff0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/lib/pq/conn_go18.go:43 +0x224 database/sql.ctxDriverExec(0x6a55220, 0xc4200180f8, 0x73e8fd8, 0xc4204c7b80, 0xc422292ff0, 0x4c, 0x6cf0d10, 0x0, 0x0, 0x402dd49, ...) /Users/andrei/work/src/go/src/database/sql/ctxutil.go:31 +0x277 database/sql.(*DB).execDC.func2() /Users/andrei/work/src/go/src/database/sql/sql.go:1253 +0x99 database/sql.withLock(0x6a3fc20, 0xc4223268c0, 0xc42202dc68) /Users/andrei/work/src/go/src/database/sql/sql.go:2855 +0x65 database/sql.(*DB).execDC(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc4223268c0, 0xc42202dd48, 0xc422292ff0, 0x4c, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1252 +0x4e4 database/sql.(*DB).exec(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422292ff0, 0x4c, 0x0, 0x0, 0x0, 0xc421945101, 0x735a6c8, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1238 +0x13e database/sql.(*DB).ExecContext(0xc421050280, 0x6a55220, 0xc4200180f8, 0xc422292ff0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0x4c, ...) /Users/andrei/work/src/go/src/database/sql/sql.go:1216 +0xe1 database/sql.(*DB).Exec(0xc421050280, 0xc422292ff0, 0x4c, 0x0, 0x0, 0x0, 0x4c, 0xc4211d96a0, 0x0, 0x0) /Users/andrei/work/src/go/src/database/sql/sql.go:1230 +0x85 github.com/cockroachdb/cockroach/pkg/sql_test.createTestTable2(0xc4202d8780, 0xc4202daaf0, 0x3b, 0x3e8, 0xc421050280, 0xc421b3e4c0, 0xc421b3e4d0, 0xc421b4a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:399 +0x37f created by github.com/cockroachdb/cockroach/pkg/sql_test.TestParallelDropCreateTables /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_test.go:347 +0x2e1 goroutine 1733 [IO wait]: internal/poll.runtime_pollWait(0x73e3dd0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c33698, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c33698, 0xc42228c400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c33680, 0xc42228c400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c33680, 0xc42228c400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8d80, 0xc42228c400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc422a97360, 0xc42228c400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420fafe90, 0x73ccbf8, 0xc422a97360, 0x5, 0xc422a97360, 0xad) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b16380, 0x5d2d117, 0xc420b164a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b16380, 0xc421b38000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc420371980, 0xc421b38000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054a3e0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054a3e0, 0xbe9de8a31c89bd88, 0x7bce326ad, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054a510, 0x6a46020, 0xc42054a3e0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054a380, 0x6a551e0, 0xc420cf0800, 0xc421229940, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420fef980, 0x6a611e0, 0xc420b16380, 0x0, 0x0, 0xc422c3400d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc420fef980, 0x6a611e0, 0xc420b16380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420fafe60, 0x6a61a80, 0xc422a97360, 0xc422beaa40, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc422a97360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc422a97360, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1762 [IO wait]: internal/poll.runtime_pollWait(0x73e4010, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c33718, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c33718, 0xc422533400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c33700, 0xc422533400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c33700, 0xc422533400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8d88, 0xc422533400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc422a97400, 0xc422533400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b22ae0, 0x73ccbf8, 0xc422a97400, 0x5, 0xc422a97400, 0xc3) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420e96380, 0x5d2d117, 0xc420e964a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420e96380, 0xc422f7e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e8fc40, 0xc422f7e000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4224d4060) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4224d4060, 0xbe9de8a32f529020, 0x7cfabd32e, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4224d4190, 0x6a46020, 0xc4224d4060, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4224d4000, 0x6a551e0, 0xc4211553c0, 0xc4215edc20, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42106fc80, 0x6a611e0, 0xc420e96380, 0x0, 0x0, 0xc421e9504a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc42106fc80, 0x6a611e0, 0xc420e96380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222cc930, 0x6a61a80, 0xc422a97400, 0xc422c10a60, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc422a97400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc422a97400, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1649 [IO wait]: internal/poll.runtime_pollWait(0x73e3d10, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420c33798, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420c33798, 0xc422533000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420c33780, 0xc422533000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420c33780, 0xc422533000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f8d90, 0xc422533000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc422a972c0, 0xc422533000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b22ab0, 0x73ccbf8, 0xc422a972c0, 0x5, 0xc422a972c0, 0xb2) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420466a80, 0x5d2d117, 0xc420466ba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420466a80, 0xc4227a5000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421ebfe20, 0xc4227a5000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc421a80220) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc421a80220, 0xbe9de8a305878ed0, 0x7a5e12489, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc421a80350, 0x6a46020, 0xc421a80220, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc421a801c0, 0x6a551e0, 0xc4212f0d40, 0xc4228741c0, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4211662c0, 0x6a611e0, 0xc420466a80, 0x0, 0x0, 0xc421e9400d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc4211662c0, 0x6a611e0, 0xc420466a80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222cc900, 0x6a61a80, 0xc422a972c0, 0xc422c10a40, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc422a972c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc422a972c0, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 114412 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc421fcd2c0, 0x45563f3, 0xc4244091e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc421fcd2c0, 0x5d2c558, 0xc424409420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42275b1d0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42275b1d0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42371ebc8, 0x5b6ec40, 0xc421a1d8c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424264900, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42371ebc8, 0x5b6ec40, 0xc421a1d8c0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424264900, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42371ebc8, 0x5b6ec40, 0xc421a1d8c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424264900, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42371ebc8, 0x5b6ec40, 0xc421a1d8c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80150, 0x6a551e0, 0xc424264900, 0xc42371ebc8, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x770c190) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422993880, 0xc42371ebb0, 0x6a551e0, 0xc424264900, 0xc400000008, 0xc424733f80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422993880, 0x6a551e0, 0xc424264900, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422993880, 0x6a551e0, 0xc424264900, 0xc42371ea50, 0xc423140c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 1791 [IO wait]: internal/poll.runtime_pollWait(0x7724540, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96298, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96298, 0xc4220c5c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96280, 0xc4220c5c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96280, 0xc4220c5c00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e420, 0xc4220c5c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4221c85a0, 0xc4220c5c00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b23cb0, 0x73ccbf8, 0xc4221c85a0, 0x5, 0xc4221c85a0, 0x9b) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f06000, 0x5d2d117, 0xc421f06120, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f06000, 0xc4232fe000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422fb8aa0, 0xc4232fe000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc421a81e20) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc421a81e20, 0xbe9de8a3231a38a8, 0x7c3739773, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc421a81f50, 0x6a46020, 0xc421a81e20, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc421a81dc0, 0x6a551e0, 0xc42302a180, 0xc423007190, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4201caf40, 0x6a611e0, 0xc421f06000, 0x0, 0x0, 0xc4216b000d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4201caf40, 0x6a611e0, 0xc421f06000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222be660, 0x6a61a80, 0xc4221c85a0, 0xc4203a3b80, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4221c85a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc4221c85a0, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1861 [IO wait]: internal/poll.runtime_pollWait(0x7724600, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96318, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96318, 0xc4224db400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96300, 0xc4224db400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96300, 0xc4224db400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e428, 0xc4224db400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4221c8640, 0xc4224db400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420f2fdd0, 0x73ccbf8, 0xc4221c8640, 0x5, 0xc4221c8640, 0xa4) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422527880, 0x5d2d117, 0xc4225279a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422527880, 0xc420d99000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422432b60, 0xc420d99000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc423226920) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc423226920, 0xbe9de8a3176e8900, 0x7b7c7fb66, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc423226a50, 0x6a46020, 0xc423226920, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4232268c0, 0x6a551e0, 0xc42104b540, 0xc420395550, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4208a5f00, 0x6a611e0, 0xc422527880, 0x0, 0x0, 0xc421e9000d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4208a5f00, 0x6a611e0, 0xc422527880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420f2fda0, 0x6a61a80, 0xc4221c8640, 0xc420351f90, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4221c8640) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc4221c8640, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1826 [IO wait]: internal/poll.runtime_pollWait(0x73ea928, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96398, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96398, 0xc422084000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96380, 0xc422084000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96380, 0xc422084000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e430, 0xc422084000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4221c86e0, 0xc422084000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b23d10, 0x73ccbf8, 0xc4221c86e0, 0x5, 0xc4221c86e0, 0xc4) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f06380, 0x5d2d117, 0xc421f064a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f06380, 0xc422e57000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e24900, 0xc422e57000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054baa0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054baa0, 0xbe9de8a3099ed848, 0x7a9f86974, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054bbd0, 0x6a46020, 0xc42054baa0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054ba40, 0x6a551e0, 0xc420feff00, 0xc422877840, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4201cb0c0, 0x6a611e0, 0xc421f06380, 0x0, 0x0, 0xc4216b100d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4201cb0c0, 0x6a611e0, 0xc421f06380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420e49830, 0x6a61a80, 0xc4221c86e0, 0xc421c370c0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4221c86e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc4221c86e0, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1827 [IO wait]: internal/poll.runtime_pollWait(0x7724480, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96418, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96418, 0xc422084800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96400, 0xc422084800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96400, 0xc422084800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e440, 0xc422084800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4221c8780, 0xc422084800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b23da0, 0x73ccbf8, 0xc4221c8780, 0x5, 0xc4221c8780, 0xb8) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f06700, 0x5d2d117, 0xc421f06820, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f06700, 0xc422f1a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422ec8800, 0xc422f1a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc421a80760) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc421a80760, 0xbe9de8a31df91498, 0x7be527d21, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc421a80890, 0x6a46020, 0xc421a80760, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc421a80700, 0x6a551e0, 0xc4212f1b80, 0xc422875490, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4201cb240, 0x6a611e0, 0xc421f06700, 0x0, 0x0, 0xc422480022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4201cb240, 0x6a611e0, 0xc421f06700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420e49a40, 0x6a61a80, 0xc4221c8780, 0xc421c370f0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4221c8780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc4221c8780, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1753 [IO wait]: internal/poll.runtime_pollWait(0x77243c0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421a96498, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421a96498, 0xc422085800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421a96480, 0xc422085800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421a96480, 0xc422085800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc42000e450, 0xc422085800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4221c8000, 0xc422085800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b23e00, 0x73ccbf8, 0xc4221c8000, 0x5, 0xc4221c8000, 0xa3) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f06e00, 0x5d2d117, 0xc421f06f20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f06e00, 0xc422fc6000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4229026a0, 0xc422fc6000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc420cd0220) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc420cd0220, 0xbe9de8a32a1139e0, 0x7ca6a8979, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc420cd0350, 0x6a46020, 0xc420cd0220, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc420cd01c0, 0x6a551e0, 0xc421154780, 0xc420394720, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4203b4240, 0x6a611e0, 0xc421f06e00, 0x0, 0x0, 0xc42248a037, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4203b4240, 0x6a611e0, 0xc421f06e00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420d253b0, 0x6a61a80, 0xc4221c8000, 0xc4211924e0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4221c8000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc4221c8000, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1803 [IO wait]: internal/poll.runtime_pollWait(0x7724300, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971298, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971298, 0xc4220cd800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971280, 0xc4220cd800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971280, 0xc4220cd800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee198, 0xc4220cd800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015eaa0, 0xc4220cd800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421012450, 0x73ccbf8, 0xc42015eaa0, 0x5, 0xc42015eaa0, 0xc4) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42197ee00, 0x5d2d117, 0xc42197ef20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42197ee00, 0xc422fba000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e8fca0, 0xc422fba000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4224d4220) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4224d4220, 0xbe9de8a326560a88, 0x7c6af626f, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4224d4350, 0x6a46020, 0xc4224d4220, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4224d41c0, 0x6a551e0, 0xc4211554c0, 0xc422e80c70, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4203b59c0, 0x6a611e0, 0xc42197ee00, 0x0, 0x0, 0xc421c2f022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc4203b59c0, 0x6a611e0, 0xc42197ee00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222c96e0, 0x6a61a80, 0xc42015eaa0, 0xc42292d300, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015eaa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015eaa0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1829 [IO wait]: internal/poll.runtime_pollWait(0x7724240, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971318, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971318, 0xc420f0ac00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971300, 0xc420f0ac00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971300, 0xc420f0ac00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1a0, 0xc420f0ac00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015ebe0, 0xc420f0ac00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421012570, 0x73ccbf8, 0xc42015ebe0, 0x5, 0xc42015ebe0, 0xb4) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224e6e00, 0x5d2d117, 0xc4224e6f20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224e6e00, 0xc422de3000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422db2e00, 0xc422de3000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054b720) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054b720, 0xbe9de8a3109d9dc8, 0x7b0f72067, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054b850, 0x6a46020, 0xc42054b720, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054b6c0, 0x6a551e0, 0xc420fef380, 0xc422876d10, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420738480, 0x6a611e0, 0xc4224e6e00, 0x0, 0x0, 0xc4220ef035, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420738480, 0x6a611e0, 0xc4224e6e00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420e49d70, 0x6a61a80, 0xc42015ebe0, 0xc421c37140, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015ebe0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015ebe0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 115110 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc420bd58c0, 0xc420518e60, 0x7, 0xc421989b80, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc420bd58c0, 0x7, 0xc421989b80, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc420bd58c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f01100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc420bd58c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f01100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc420bd58c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f01100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc420bd58c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc421acea80, 0x6a552a0, 0xc422752900, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc421acea80, 0x6a552a0, 0xc422752900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc422752630) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc422752630, 0xc422266f90, 0x29, 0x6a652e0, 0xc420358700, 0xc42164f4b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 1805 [IO wait]: internal/poll.runtime_pollWait(0x7724180, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971418, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971418, 0xc420710000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971400, 0xc420710000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971400, 0xc420710000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1b0, 0xc420710000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015ed20, 0xc420710000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210124b0, 0x73ccbf8, 0xc42015ed20, 0x5, 0xc42015ed20, 0xc8) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42197f880, 0x5d2d117, 0xc42197f9a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42197f880, 0xc422e85000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421fd6400, 0xc422e85000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc423226060) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc423226060, 0xbe9de8a310b5b5c0, 0x7b10f3709, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc423226190, 0x6a46020, 0xc423226060, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc423226000, 0x6a551e0, 0xc42132b080, 0xc4232012c0, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420061900, 0x6a611e0, 0xc42197f880, 0x0, 0x0, 0xc421c3300d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420061900, 0x6a611e0, 0xc42197f880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222c9770, 0x6a61a80, 0xc42015ed20, 0xc42292d350, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015ed20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015ed20, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1806 [IO wait]: internal/poll.runtime_pollWait(0x7723f40, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971498, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971498, 0xc420710400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971480, 0xc420710400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971480, 0xc420710400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1b8, 0xc420710400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015edc0, 0xc420710400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210124e0, 0x73ccbf8, 0xc42015edc0, 0x5, 0xc42015edc0, 0xbf) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224e6000, 0x5d2d117, 0xc4224e6120, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224e6000, 0xc421946000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc423205f60, 0xc421946000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc93a0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc93a0, 0xbe9de8a33084caa8, 0x7d0de0c3c, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc94d0, 0x6a46020, 0xc422fc93a0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc9340, 0x6a551e0, 0xc423229900, 0xc423200970, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420061bc0, 0x6a611e0, 0xc4224e6000, 0x0, 0x0, 0xc421fc400d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420061bc0, 0x6a611e0, 0xc4224e6000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222c97a0, 0x6a61a80, 0xc42015edc0, 0xc42292d370, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015edc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015edc0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1860 [IO wait]: internal/poll.runtime_pollWait(0x7723e80, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971518, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971518, 0xc422054c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971500, 0xc422054c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971500, 0xc422054c00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1c0, 0xc422054c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015ee60, 0xc422054c00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420f2f950, 0x73ccbf8, 0xc42015ee60, 0x5, 0xc42015ee60, 0x9f) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420b17880, 0x5d2d117, 0xc420b179a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420b17880, 0xc422fbb000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421fd6880, 0xc422fbb000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4232263e0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4232263e0, 0xbe9de8a30a751408, 0x7aacea290, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc423226510, 0x6a46020, 0xc4232263e0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc423226380, 0x6a551e0, 0xc42132b800, 0xc423201420, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4208a54c0, 0x6a611e0, 0xc420b17880, 0x0, 0x0, 0xc4224c6022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc4208a54c0, 0x6a611e0, 0xc420b17880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420f2f920, 0x6a61a80, 0xc42015ee60, 0xc420351e40, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015ee60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015ee60, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1830 [IO wait]: internal/poll.runtime_pollWait(0x7724000, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971598, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971598, 0xc4222dfc00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971580, 0xc4222dfc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971580, 0xc4222dfc00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1c8, 0xc4222dfc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015ef00, 0xc4222dfc00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420faea80, 0x73ccbf8, 0xc42015ef00, 0x5, 0xc42015ef00, 0xc9) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c58000, 0x5d2d117, 0xc421c58120, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c58000, 0xc422232000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421209300, 0xc422232000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc420cd1720) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc420cd1720, 0xbe9de8a30a1aede8, 0x7aa747d6b, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc420cd1850, 0x6a46020, 0xc420cd1720, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc420cd16c0, 0x6a551e0, 0xc42101d000, 0xc4218a2950, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42101d540, 0x6a611e0, 0xc421c58000, 0x0, 0x0, 0xc4226cc037, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc42101d540, 0x6a611e0, 0xc421c58000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420faea50, 0x6a61a80, 0xc42015ef00, 0xc421c37e50, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015ef00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015ef00, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1793 [IO wait]: internal/poll.runtime_pollWait(0x7723dc0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971618, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971618, 0xc420f0a800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971600, 0xc420f0a800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971600, 0xc420f0a800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1d8, 0xc420f0a800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015efa0, 0xc420f0a800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421012540, 0x73ccbf8, 0xc42015efa0, 0x5, 0xc42015efa0, 0xa6) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224e6a80, 0x5d2d117, 0xc4224e6ba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224e6a80, 0xc422e84000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e25ba0, 0xc422e84000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054be20) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054be20, 0xbe9de8a327ac4fa0, 0x7c805a527, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054bf50, 0x6a46020, 0xc42054be20, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054bdc0, 0x6a551e0, 0xc421154340, 0xc422e80020, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420061f80, 0x6a611e0, 0xc4224e6a80, 0x0, 0x0, 0xc4220ee04a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420061f80, 0x6a611e0, 0xc4224e6a80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420e491a0, 0x6a61a80, 0xc42015efa0, 0xc421c370a0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015efa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015efa0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1814 [IO wait]: internal/poll.runtime_pollWait(0x7723d00, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971798, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971798, 0xc420f0bc00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971780, 0xc420f0bc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971780, 0xc420f0bc00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1e0, 0xc420f0bc00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f040, 0xc420f0bc00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210125d0, 0x73ccbf8, 0xc42015f040, 0x5, 0xc42015f040, 0xc0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224e7880, 0x5d2d117, 0xc4224e79a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224e7880, 0xc422e43000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e24620, 0xc422e43000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42054b8e0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42054b8e0, 0xbe9de8a31f9e2978, 0x7bff78cde, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42054ba10, 0x6a46020, 0xc42054b8e0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42054b880, 0x6a551e0, 0xc420fefc00, 0xc422877750, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420738a00, 0x6a611e0, 0xc4224e7880, 0x0, 0x0, 0xc421fc9037, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420738a00, 0x6a611e0, 0xc4224e7880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc42226d830, 0x6a61a80, 0xc42015f040, 0xc4222c5b80, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015f040, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1832 [IO wait]: internal/poll.runtime_pollWait(0x7723c40, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971818, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971818, 0xc4210f7400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971800, 0xc4210f7400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971800, 0xc4210f7400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee1e8, 0xc4210f7400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f0e0, 0xc4210f7400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420faf440, 0x73ccbf8, 0xc42015f0e0, 0x5, 0xc42015f0e0, 0xa2) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c59500, 0x5d2d117, 0xc421c59620, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c59500, 0xc423076000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc42305a1e0, 0xc423076000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc83e0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc83e0, 0xbe9de8a3203bc908, 0x7c0952b40, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc8510, 0x6a46020, 0xc422fc83e0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc8380, 0x6a551e0, 0xc42101be80, 0xc422867ad0, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42101d740, 0x6a611e0, 0xc421c59500, 0x0, 0x0, 0xc421f0204a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc42101d740, 0x6a611e0, 0xc421c59500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420faf410, 0x6a61a80, 0xc42015f0e0, 0xc420374ac0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f0e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc42015f0e0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 114873 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423773770, 0xc4206a4230, 0x7, 0xc423081770, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423773770, 0x7, 0xc423081770, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423773770, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423773770, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423773770, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423773770, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4202d0000, 0x6a552a0, 0xc423773740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc425248f78, 0xc4227ef100, 0x4, 0xc423640940) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4227ef100, 0x5, 0xc423640900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x3a, 0x3a, 0x15179d571b2d6c48, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x3a, 0x3a, 0x15179d571b2d6c48, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42375f380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 1807 [IO wait]: internal/poll.runtime_pollWait(0x7725980, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971918, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971918, 0xc422533800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971900, 0xc422533800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971900, 0xc422533800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee258, 0xc422533800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f180, 0xc422533800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b22b10, 0x73ccbf8, 0xc42015f180, 0x5, 0xc42015f180, 0xb3) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420e96a80, 0x5d2d117, 0xc420e96ba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420e96a80, 0xc422e45000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e266e0, 0xc422e45000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42117d3a0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42117d3a0, 0xbe9de8a32b404bd0, 0x7cb9997e4, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42117d4d0, 0x6a46020, 0xc42117d3a0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42117d340, 0x6a551e0, 0xc42104b800, 0xc4215ed290, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42106fd00, 0x6a611e0, 0xc420e96a80, 0x0, 0x0, 0xc42278c035, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc42106fd00, 0x6a611e0, 0xc420e96a80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222c97d0, 0x6a61a80, 0xc42015f180, 0xc42292d3a0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc42015f180, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1808 [IO wait]: internal/poll.runtime_pollWait(0x7723ac0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971998, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971998, 0xc422533c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971980, 0xc422533c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971980, 0xc422533c00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee268, 0xc422533c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f220, 0xc422533c00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b22b40, 0x73ccbf8, 0xc42015f220, 0x5, 0xc42015f220, 0xb5) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc420e97c00, 0x5d2d117, 0xc420e97d20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc420e97c00, 0xc422f30000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422f063a0, 0xc422f30000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42117de20) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42117de20, 0xbe9de8a325d2dc80, 0x7c62c3485, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42117df50, 0x6a46020, 0xc42117de20, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42117ddc0, 0x6a551e0, 0xc42104bfc0, 0xc4215edf70, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42106fe00, 0x6a611e0, 0xc420e97c00, 0x0, 0x0, 0xc42278d022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc42106fe00, 0x6a611e0, 0xc420e97c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222c9800, 0x6a61a80, 0xc42015f220, 0xc42292d3c0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f220) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc42015f220, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1809 [IO wait]: internal/poll.runtime_pollWait(0x77258c0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971a18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971a18, 0xc42278a000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971a00, 0xc42278a000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971a00, 0xc42278a000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee270, 0xc42278a000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f2c0, 0xc42278a000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b22b70, 0x73ccbf8, 0xc42015f2c0, 0x5, 0xc42015f2c0, 0xa5) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4210d7180, 0x5d2d117, 0xc4210d72a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4210d7180, 0xc42312d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422b1ee80, 0xc42312d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4224d4e60) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4224d4e60, 0xbe9de8a303984da8, 0x7a3f1e9ef, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4224d4f90, 0x6a46020, 0xc4224d4e60, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4224d4e00, 0x6a551e0, 0xc422bd8180, 0xc422e81160, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42106ff80, 0x6a611e0, 0xc4210d7180, 0x0, 0x0, 0xc422520020, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc42106ff80, 0x6a611e0, 0xc4210d7180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222c9830, 0x6a61a80, 0xc42015f2c0, 0xc42292d3f0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc42015f2c0, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1810 [IO wait]: internal/poll.runtime_pollWait(0x7725800, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971a98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971a98, 0xc42278a400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971a80, 0xc42278a400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971a80, 0xc42278a400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee280, 0xc42278a400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f360, 0xc42278a400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b22bd0, 0x73ccbf8, 0xc42015f360, 0x5, 0xc42015f360, 0xae) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42113c380, 0x5d2d117, 0xc42113c4a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42113c380, 0xc42189f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422eb2540, 0xc42189f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fd2760) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fd2760, 0xbe9de8a317de0410, 0x7b8377736, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fd2890, 0x6a46020, 0xc422fd2760, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fd2700, 0x6a551e0, 0xc423228380, 0xc422f41e20, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4211bb000, 0x6a611e0, 0xc42113c380, 0x0, 0x0, 0xc42252100d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc4211bb000, 0x6a611e0, 0xc42113c380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222c9860, 0x6a61a80, 0xc42015f360, 0xc42292d410, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc42015f360, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1812 [IO wait]: internal/poll.runtime_pollWait(0x7725680, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971b98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971b98, 0xc42223ac00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971b80, 0xc42223ac00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971b80, 0xc42223ac00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee290, 0xc42223ac00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f4a0, 0xc42223ac00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b22c30, 0x73ccbf8, 0xc42015f4a0, 0x5, 0xc42015f4a0, 0xc3) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42113dc00, 0x5d2d117, 0xc42113dd20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42113dc00, 0xc422727000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422fb91a0, 0xc422727000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4224d5c60) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4224d5c60, 0xbe9de8a310ab41f8, 0x7b104c188, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4224d5d90, 0x6a46020, 0xc4224d5c60, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4224d5c00, 0x6a551e0, 0xc42302b400, 0xc4230066e0, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4211bb8c0, 0x6a611e0, 0xc42113dc00, 0x0, 0x0, 0xc422525022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc4211bb8c0, 0x6a611e0, 0xc42113dc00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4222c98c0, 0x6a61a80, 0xc42015f4a0, 0xc42292d470, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f4a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc42015f4a0, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 115251 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423874780, 0x45563f3, 0xc4234b31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423874780, 0x5d2c558, 0xc4234b3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc421064870, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc421064870, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b30f38, 0x5b6ec40, 0xc42363afc0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423a47080, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b30f38, 0x5b6ec40, 0xc42363afc0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc423a47080, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b30f38, 0x5b6ec40, 0xc42363afc0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423a47080, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b30f38, 0x5b6ec40, 0xc42363afc0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80748, 0x6a551e0, 0xc423a47080, 0xc423b30f38, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7e4894a9b) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e74ee0, 0xc423b30f20, 0x6a551e0, 0xc423a47080, 0xc400000008, 0xc42160f2c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e74ee0, 0x6a551e0, 0xc423a47080, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e74ee0, 0x6a551e0, 0xc423a47080, 0xc423b30dc0, 0xc42363af00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 1813 [IO wait]: internal/poll.runtime_pollWait(0x7725500, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971c98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971c98, 0xc42223b400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971c80, 0xc42223b400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971c80, 0xc42223b400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee2a0, 0xc42223b400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f5e0, 0xc42223b400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b22c90, 0x73ccbf8, 0xc42015f5e0, 0x5, 0xc42015f5e0, 0xb2) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42124b180, 0x5d2d117, 0xc42124b2a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42124b180, 0xc4229c5000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4232059c0, 0xc4229c5000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc8e60) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc8e60, 0xbe9de8a32c359c98, 0x7cc8ee758, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc8f90, 0x6a46020, 0xc422fc8e60, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc8e00, 0x6a551e0, 0xc4232294c0, 0xc423200710, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4200514c0, 0x6a611e0, 0xc42124b180, 0x0, 0x0, 0xc422531035, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc4200514c0, 0x6a611e0, 0xc42124b180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc42226d800, 0x6a61a80, 0xc42015f5e0, 0xc4222c5b30, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f5e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc42015f5e0, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1831 [IO wait]: internal/poll.runtime_pollWait(0x7725440, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421971d18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421971d18, 0xc4210f6800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421971d00, 0xc4210f6800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421971d00, 0xc4210f6800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee2a8, 0xc4210f6800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42015f680, 0xc4210f6800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420faf3e0, 0x73ccbf8, 0xc42015f680, 0x5, 0xc42015f680, 0xa8) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421c59180, 0x5d2d117, 0xc421c592a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421c59180, 0xc42272b000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4232057c0, 0xc42272b000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc8ae0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc8ae0, 0xbe9de8a324a45730, 0x7c4fdb23b, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc8c10, 0x6a46020, 0xc422fc8ae0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc8a80, 0x6a551e0, 0xc423229080, 0xc4232005c0, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42101d6c0, 0x6a611e0, 0xc421c59180, 0x0, 0x0, 0xc421ed6035, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc42101d6c0, 0x6a611e0, 0xc421c59180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420faf3b0, 0x6a61a80, 0xc42015f680, 0xc420374a40, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42015f680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc42015f680, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1912 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4211e1a70, 0xc4206a4230, 0x7, 0xc424783900, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4211e1a70, 0x7, 0xc424783900, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4211e1a70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9cf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4211e1a70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9cf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4211e1a70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9cf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4211e1a70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ace000, 0x6a552a0, 0xc4211e19e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc422cee500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e9ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422342b60, 0x6a551e0, 0xc422cee500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc422342b60, 0x6a551e0, 0xc422cee500, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc422342b60, 0x6a551e0, 0xc422cee500, 0xc422342b60, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc422342b60, 0x6a551e0, 0xc422cee500, 0xc420b13bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422e4f000, 0x6a551e0, 0xc422cee500, 0x6a568e0, 0xc423419f80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422e4f000, 0x6a551e0, 0xc422cee500, 0x6a568e0, 0xc423419f80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422e4f000, 0x6a551e0, 0xc420feff40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc420feff40, 0x0, 0x0, 0xc4216b100d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422877870, 0xc42100a460, 0x6a551e0, 0xc420feff40, 0xc42054ba40, 0x5400, 0x15000, 0xc4206d5810, 0xc421c41890, 0xc422877860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114531 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422ed2a80, 0x15179d5713456fa8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422ed2a80, 0x15179d5713456fa8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422ed2a80, 0x15179d5713456fa8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422ed2a50, 0x15179d5713456fa8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422ed2960, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422ed2960, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422ed2960, 0x5c4d11c, 0x10, 0xc423e499b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422ed2960, 0xc4231d2fe8, 0xc422ed2960, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422ed2960, 0xc4231d2fe8, 0x1f, 0xc420bcb630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42311d500, 0xc4231d2fd0, 0x6a551e0, 0xc4231b7c40, 0xc400000008, 0xc423306d80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42311d500, 0x6a551e0, 0xc4231b7c40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42311d500, 0x6a551e0, 0xc4231b7c40, 0xc4231d2f20, 0xc423c5dbc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114372 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422707170, 0x15179d5710d15e80, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a4a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422707170, 0x15179d5710d15e80, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a4a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422707170, 0x15179d5710d15e80, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a4a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4227070e0, 0x15179d5710d15e80, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a4a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422707080, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a4a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422707080, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422707050, 0x5c4d11c, 0x10, 0xc423c112c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422707050, 0xc421e9c4d0, 0xc422707050, 0xc422f47568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422707050, 0xc421e9c4d0, 0xc420b53500, 0xc4241ca068, 0xc423a46540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc422706f00, 0x5c13340, 0xc421e9c4d0, 0x5c13340, 0xc421e9c4d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc422706f00, 0x5c13340, 0xc421e9c4d0, 0xc4223d6500, 0xc4223d6520, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc422706f00, 0x5c13340, 0xc421e9c4d0, 0xc4223d6500, 0xc4223d6520, 0x6, 0xc422f47968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc422706f00, 0xc421e9c460, 0xc420eaab00, 0x0, 0x0, 0x2, 0xc420dc5c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422f60780, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422f60780, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422f60780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 1836 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423720b40, 0xc4206a4230, 0x7, 0xc422b7a2d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423720b40, 0x7, 0xc422b7a2d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423720b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421c06100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423720b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421c06100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423720b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421c06100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423720b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421c06000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e08380, 0x6a552a0, 0xc423720ae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421c06000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc42160f440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421c06000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422a59520, 0x6a551e0, 0xc42160f440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc42160f440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc42160f440, 0xc423267878, 0xc424863100, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422a59520, 0x6a551e0, 0xc42160f440, 0xc424863100, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc421b6e3d0, 0x6a551e0, 0xc42160f440, 0xc421ecfe40, 0x13, 0x20, 0xcce, 0x6a651a0, 0xc421266900, 0xc42216dbc0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc422427620, 0x6a551e0, 0xc42160f440, 0xc421b6e440, 0xc421b6e3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc422427620, 0xc421be67a8, 0xc423268b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42160fb00, 0x5c46757, 0xc, 0x6a4c120, 0xc422427620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42160fb00, 0x6a4c120, 0xc422427620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc42160f440, 0x6a4c120, 0xc422427620, 0x5d28ea0, 0x0, 0x0, 0xc422427650, 0xc422eb3fc0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc42160f440, 0xc421b6e440, 0xc421b6e3d0, 0x6a4c120, 0xc422427620, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc42160f440, 0xc421b6e440, 0xc421b6e3d0, 0x6a4c120, 0xc422427620, 0x0, 0xc422eb3fc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc421b6e648, 0x6a551e0, 0xc42160f440, 0xc421b6e440, 0xc421b6e3d0, 0xddd1abfc6216b4ef, 0xc423269590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc421b6e000, 0x6a551e0, 0xc42160f440, 0xc421b6e3d0, 0x1, 0x73e91b0, 0xc4224ee1b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc421b6e000, 0x6a551e0, 0xc42160f440, 0x6a568e0, 0xc4244d8540, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc421b6e000, 0x6a551e0, 0xc42160f440, 0x6a568e0, 0xc4244d8540, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421b6e000, 0x6a551e0, 0xc42160f440, 0x6a568e0, 0xc4244d8540, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421b6e000, 0x6a551e0, 0xc4212f0040, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc4212f0040, 0x0, 0x0, 0xc422c60022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422171260, 0xc42100a460, 0x6a551e0, 0xc4212f0040, 0xc421a80000, 0x5400, 0x15000, 0xc4206d5810, 0xc4226eeb70, 0xc422171250) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1911 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc422706b10, 0xc420518e60, 0x7, 0xc420c6c7d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc422706b10, 0x7, 0xc420c6c7d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc422706b10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a4a100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc422706b10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a4a100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc422706b10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a4a100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc422706b10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a4a000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42054b180, 0x6a552a0, 0xc422706a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a4a000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc423ef81c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a4a000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4210d8ea0, 0x6a551e0, 0xc423ef81c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc423ef81c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc423ef81c0, 0xc422f97878, 0xc424b8e700, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc4210d8ea0, 0x6a551e0, 0xc423ef81c0, 0xc424b8e700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422e4e3d0, 0x6a551e0, 0xc423ef81c0, 0xc42250ebe0, 0x13, 0x20, 0xcb3, 0x6a651a0, 0xc423944000, 0xc422a42490, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc422752510, 0x6a551e0, 0xc423ef81c0, 0xc422e4e440, 0xc422e4e3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc422752510, 0xc421be6c28, 0xc422f98b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42141d580, 0x5c46757, 0xc, 0x6a4c120, 0xc422752510) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42141d580, 0x6a4c120, 0xc422752510) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc423ef81c0, 0x6a4c120, 0xc422752510, 0x5d28ea0, 0x0, 0x0, 0xc422752540, 0xc4215e3320, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc423ef81c0, 0xc422e4e440, 0xc422e4e3d0, 0x6a4c120, 0xc422752510, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc423ef81c0, 0xc422e4e440, 0xc422e4e3d0, 0x6a4c120, 0xc422752510, 0x0, 0xc4215e3320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422e4e648, 0x6a551e0, 0xc423ef81c0, 0xc422e4e440, 0xc422e4e3d0, 0x725e5fcf36c6dd40, 0xc422f99590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422e4e000, 0x6a551e0, 0xc423ef81c0, 0xc422e4e3d0, 0x1, 0x73e91b0, 0xc423eeabd0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422e4e000, 0x6a551e0, 0xc423ef81c0, 0x6a568e0, 0xc4219e5020, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422e4e000, 0x6a551e0, 0xc423ef81c0, 0x6a568e0, 0xc4219e5020, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422e4e000, 0x6a551e0, 0xc423ef81c0, 0x6a568e0, 0xc4219e5020, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422e4e000, 0x6a551e0, 0xc420fefc40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc420fefc40, 0x0, 0x0, 0xc421fc9037, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422877780, 0xc42100a000, 0x6a551e0, 0xc420fefc40, 0xc42054b880, 0x5400, 0x15000, 0xc42136a190, 0xc421c41700, 0xc422877770) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 113981 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422fb5d70, 0xc4206a4230, 0x7, 0xc422286b40, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422fb5d70, 0x7, 0xc422286b40, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422fb5d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422fb5d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422fb5d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422fb5d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42117da40, 0x6a552a0, 0xc422fb5d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc422ce9d00, 0xc42484bc00, 0x0, 0xc422ce9e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42484bc00, 0x1, 0xc422fb5cb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42229aea0, 0xc423945b00, 0x5, 0x5, 0x15179d5705cfe128, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42230f080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 1772 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421117020, 0xc4206a4230, 0x7, 0xc42128eb90, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421117020, 0x7, 0xc42128eb90, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421117020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2bf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421117020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2bf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421117020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2bf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421117020, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2bc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4230736c0, 0x6a552a0, 0xc421116c60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2bc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc42335b880, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421e2bc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422628000, 0x6a551e0, 0xc42335b880, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc422628000, 0x6a551e0, 0xc42335b880, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc422628000, 0x6a551e0, 0xc42335b880, 0xc422628000, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc422628000, 0x6a551e0, 0xc42335b880, 0xc4235e9bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc421ff5000, 0x6a551e0, 0xc42335b880, 0x6a568e0, 0xc42186e600, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421ff5000, 0x6a551e0, 0xc42335b880, 0x6a568e0, 0xc42186e600, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421ff5000, 0x6a551e0, 0xc420cf1a40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc420cf1a40, 0x0, 0x0, 0xc422d3c00d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc421229fb0, 0xc42100a460, 0x6a551e0, 0xc420cf1a40, 0xc42054ae00, 0x5400, 0x15000, 0xc4206d5810, 0xc420413090, 0xc421229fa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1852 [IO wait]: internal/poll.runtime_pollWait(0x938f3d8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712898, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712898, 0xc422c1e000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712880, 0xc422c1e000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712880, 0xc422c1e000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee560, 0xc422c1e000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421050fa0, 0xc422c1e000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421013560, 0x73ccbf8, 0xc421050fa0, 0x5, 0xc421050fa0, 0xaa) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfa000, 0x5d2d117, 0xc422bfa120, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfa000, 0xc423044000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422b1e6c0, 0xc423044000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4224d4ae0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4224d4ae0, 0xbe9de8a31bf26f28, 0x7bc4bd933, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4224d4c10, 0x6a46020, 0xc4224d4ae0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4224d4a80, 0x6a551e0, 0xc421155d40, 0xc422e80f40, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420739200, 0x6a611e0, 0xc422bfa000, 0x0, 0x0, 0xc422b7f00d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420739200, 0x6a611e0, 0xc422bfa000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421013500, 0x6a61a80, 0xc421050fa0, 0xc420ce9cf0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421050fa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc421050fa0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1853 [IO wait]: internal/poll.runtime_pollWait(0x938f318, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712a98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712a98, 0xc422c1e400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712a80, 0xc422c1e400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712a80, 0xc422c1e400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee568, 0xc422c1e400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421051040, 0xc422c1e400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210135f0, 0x73ccbf8, 0xc421051040, 0x5, 0xc421051040, 0xc7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfa380, 0x5d2d117, 0xc422bfa4a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfa380, 0xc422ee7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421fd6640, 0xc422ee7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc423226220) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc423226220, 0xbe9de8a322a35ee0, 0x7c2fcbebb, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc423226350, 0x6a46020, 0xc423226220, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4232261c0, 0x6a551e0, 0xc42132b440, 0xc423201360, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420739480, 0x6a611e0, 0xc422bfa380, 0x0, 0x0, 0xc422c3100d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420739480, 0x6a611e0, 0xc422bfa380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421013590, 0x6a61a80, 0xc421051040, 0xc420ce9d40, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421051040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc421051040, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1854 [IO wait]: internal/poll.runtime_pollWait(0x938f258, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712b18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712b18, 0xc422c1e800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712b00, 0xc422c1e800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712b00, 0xc422c1e800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee570, 0xc422c1e800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4210510e0, 0xc422c1e800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421013740, 0x73ccbf8, 0xc4210510e0, 0x5, 0xc4210510e0, 0xb7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfa700, 0x5d2d117, 0xc422bfa820, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfa700, 0xc422f18000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422ec8780, 0xc422f18000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc421a805a0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc421a805a0, 0xbe9de8a30ac72ba8, 0x7ab20baa2, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc421a806d0, 0x6a46020, 0xc421a805a0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc421a80540, 0x6a551e0, 0xc4212f1a80, 0xc422875430, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420739600, 0x6a611e0, 0xc422bfa700, 0x0, 0x0, 0xc420cf904a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420739600, 0x6a611e0, 0xc422bfa700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421013620, 0x6a61a80, 0xc4210510e0, 0xc420ce9d70, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4210510e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc4210510e0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1846 [IO wait]: internal/poll.runtime_pollWait(0x938f198, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712b98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712b98, 0xc420534000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712b80, 0xc420534000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712b80, 0xc420534000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee578, 0xc420534000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421050e60, 0xc420534000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210128a0, 0x73ccbf8, 0xc421050e60, 0x5, 0xc421050e60, 0xab) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224e7c00, 0x5d2d117, 0xc4224e7d20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224e7c00, 0xc420ed2000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc423205f20, 0xc420ed2000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc91e0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc91e0, 0xbe9de8a30d18ef90, 0x7ad727721, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc9310, 0x6a46020, 0xc422fc91e0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc9180, 0x6a551e0, 0xc423229840, 0xc423200920, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420738d80, 0x6a611e0, 0xc4224e7c00, 0x0, 0x0, 0xc4222c104a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc420738d80, 0x6a611e0, 0xc4224e7c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421012810, 0x6a61a80, 0xc421050e60, 0xc420ce89c0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421050e60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc421050e60, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1855 [IO wait]: internal/poll.runtime_pollWait(0x938f018, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712c98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712c98, 0xc422c1ec00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712c80, 0xc422c1ec00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712c80, 0xc422c1ec00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee580, 0xc422c1ec00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421051180, 0xc422c1ec00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421013860, 0x73ccbf8, 0xc421051180, 0x5, 0xc421051180, 0xb1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfaa80, 0x5d2d117, 0xc422bfaba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfaa80, 0xc4222c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422fb8b40, 0xc4222c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4224d4760) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4224d4760, 0xbe9de8a32448c9b0, 0x7c4a22311, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4224d4890, 0x6a46020, 0xc4224d4760, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4224d4700, 0x6a551e0, 0xc42302a940, 0xc423006440, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420739780, 0x6a611e0, 0xc422bfaa80, 0x0, 0x0, 0xc420cfb037, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc420739780, 0x6a611e0, 0xc422bfaa80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421013770, 0x6a61a80, 0xc421051180, 0xc420ce9de0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421051180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc421051180, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1856 [IO wait]: internal/poll.runtime_pollWait(0x938f0d8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712d18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712d18, 0xc422c1f000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712d00, 0xc422c1f000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712d00, 0xc422c1f000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee588, 0xc422c1f000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421051360, 0xc422c1f000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4210139e0, 0x73ccbf8, 0xc421051360, 0x5, 0xc421051360, 0xc1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfae00, 0x5d2d117, 0xc422bfaf20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfae00, 0xc422d76000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4232053a0, 0xc422d76000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42117cae0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42117cae0, 0xbe9de8a31d51ad20, 0x7bdab17ca, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42117cc10, 0x6a46020, 0xc42117cae0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42117ca80, 0x6a551e0, 0xc423228c00, 0xc423200410, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420739900, 0x6a611e0, 0xc422bfae00, 0x0, 0x0, 0xc421309035, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc420739900, 0x6a611e0, 0xc422bfae00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421013950, 0x6a61a80, 0xc421051360, 0xc420ce9e10, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421051360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc421051360, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1857 [IO wait]: internal/poll.runtime_pollWait(0x938ef58, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712d98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712d98, 0xc422c1f400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712d80, 0xc422c1f400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712d80, 0xc422c1f400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee590, 0xc422c1f400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421051540, 0xc422c1f400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421013ad0, 0x73ccbf8, 0xc421051540, 0x5, 0xc421051540, 0xc7) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfb180, 0x5d2d117, 0xc422bfb2a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfb180, 0xc422c5b000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc423205640, 0xc422c5b000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42117dc60) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42117dc60, 0xbe9de8a30fbd4c00, 0x7b016cea6, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42117dd90, 0x6a46020, 0xc42117dc60, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42117dc00, 0x6a551e0, 0xc423228ec0, 0xc423200520, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420739a80, 0x6a611e0, 0xc422bfb180, 0x0, 0x0, 0xc420e9900d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc420739a80, 0x6a611e0, 0xc422bfb180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421013aa0, 0x6a61a80, 0xc421051540, 0xc420ce9e50, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421051540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc421051540, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1874 [IO wait]: internal/poll.runtime_pollWait(0x938ee98, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712e18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712e18, 0xc422c1f800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712e00, 0xc422c1f800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712e00, 0xc422c1f800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee5a0, 0xc422c1f800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4210515e0, 0xc422c1f800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc421013bf0, 0x73ccbf8, 0xc4210515e0, 0x5, 0xc4210515e0, 0x9e) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422bfb500, 0x5d2d117, 0xc422bfb620, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422bfb500, 0xc422fc7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422e7ac00, 0xc422fc7000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc8060) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc8060, 0xbe9de8a3078a1590, 0x7a7e3a85f, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc8190, 0x6a46020, 0xc422fc8060, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc8000, 0x6a551e0, 0xc42101b9c0, 0xc422867280, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420739dc0, 0x6a611e0, 0xc422bfb500, 0x0, 0x0, 0xc420e9b00d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc420739dc0, 0x6a611e0, 0xc422bfb500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421013b60, 0x6a61a80, 0xc4210515e0, 0xc420ce9e90, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4210515e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc4210515e0, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1864 [IO wait]: internal/poll.runtime_pollWait(0x938edd8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420712f18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420712f18, 0xc421694000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420712f00, 0xc421694000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420712f00, 0xc421694000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ee5c0, 0xc421694000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421050f00, 0xc421694000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420b7e690, 0x73ccbf8, 0xc421050f00, 0x5, 0xc421050f00, 0xa1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421bfb180, 0x5d2d117, 0xc421bfb2a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421bfb180, 0xc42194f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc423205840, 0xc42194f000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fc8ca0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fc8ca0, 0xbe9de8a3285b40c8, 0x7c8b494d8, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fc8dd0, 0x6a46020, 0xc422fc8ca0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fc8c40, 0x6a551e0, 0xc4232292c0, 0xc423200640, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420bb8f00, 0x6a611e0, 0xc421bfb180, 0x0, 0x0, 0xc421691022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc420bb8f00, 0x6a611e0, 0xc421bfb180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420b7e660, 0x6a61a80, 0xc421050f00, 0xc4212b0c40, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421050f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc421050f00, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1858 [IO wait]: internal/poll.runtime_pollWait(0x938eb98, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208ba598, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208ba598, 0xc421b0e400, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208ba580, 0xc421b0e400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208ba580, 0xc421b0e400, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f83e8, 0xc421b0e400, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420f44640, 0xc421b0e400, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420f2f500, 0x73ccbf8, 0xc420f44640, 0x5, 0xc420f44640, 0xb9) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4226f7880, 0x5d2d117, 0xc4226f79a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4226f7880, 0xc422f08000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422ec8700, 0xc422f08000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc421a803e0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc421a803e0, 0xbe9de8a321558220, 0x7c1aee3db, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc421a80510, 0x6a46020, 0xc421a803e0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc421a80380, 0x6a551e0, 0xc4212f19c0, 0xc4228753d0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4208a4d00, 0x6a611e0, 0xc4226f7880, 0x0, 0x0, 0xc42272a020, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4208a4d00, 0x6a611e0, 0xc4226f7880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420f2f4d0, 0x6a61a80, 0xc420f44640, 0xc420351d80, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc420f44640) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc420f44640, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1859 [IO wait]: internal/poll.runtime_pollWait(0x938ead8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208ba618, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208ba618, 0xc421b0e800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208ba600, 0xc421b0e800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208ba600, 0xc421b0e800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f83f0, 0xc421b0e800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420f44820, 0xc421b0e800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420f2f5f0, 0x73ccbf8, 0xc420f44820, 0x5, 0xc420f44820, 0xac) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4226f7c00, 0x5d2d117, 0xc4226f7d20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4226f7c00, 0xc422c1b000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc42048f2c0, 0xc422c1b000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42293a060) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42293a060, 0xbe9de8a330277420, 0x7d080b8b1, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42293a190, 0x6a46020, 0xc42293a060, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42293a000, 0x6a551e0, 0xc4229b6200, 0xc4203786d0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc4208a4f80, 0x6a611e0, 0xc4226f7c00, 0x0, 0x0, 0xc42294000d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc4208a4f80, 0x6a611e0, 0xc4226f7c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420f2f5c0, 0x6a61a80, 0xc420f44820, 0xc420351dd0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc420f44820) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc420f44820, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1759 [IO wait]: internal/poll.runtime_pollWait(0x938ea18, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4208ba698, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4208ba698, 0xc42259c800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4208ba680, 0xc42259c800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4208ba680, 0xc42259c800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f83f8, 0xc42259c800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420f44280, 0xc42259c800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc420f2eab0, 0x73ccbf8, 0xc420f44280, 0x5, 0xc420f44280, 0xbd) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4226f6000, 0x5d2d117, 0xc4226f6120, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4226f6000, 0xc42307a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421cb05a0, 0xc42307a000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422fd2220) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422fd2220, 0xbe9de8a3224f5ef8, 0x7c2a8bd38, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422fd2350, 0x6a46020, 0xc422fd2220, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422fd21c0, 0x6a551e0, 0xc42132b100, 0xc422f415b0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42119ffc0, 0x6a611e0, 0xc4226f6000, 0x0, 0x0, 0xc4224ca022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc42119ffc0, 0x6a611e0, 0xc4226f6000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc420f2ea80, 0x6a61a80, 0xc420f44280, 0xc420351080, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc420f44280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc420f44280, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1905 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc422b21590, 0xc4202d8b90, 0x7, 0xc4243e0780, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc422b21590, 0x7, 0xc4243e0780, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc422b21590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6b400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc422b21590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6b400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc422b21590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6b400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc422b21590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6b300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423072a80, 0x6a552a0, 0xc422b21530, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6b300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc4241ccd00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6b300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc420d2cb60, 0x6a551e0, 0xc4241ccd00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc420d2cb60, 0x6a551e0, 0xc4241ccd00, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc420d2cb60, 0x6a551e0, 0xc4241ccd00, 0xc420d2cb60, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc420d2cb60, 0x6a551e0, 0xc4241ccd00, 0xc424ad5bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422d87000, 0x6a551e0, 0xc4241ccd00, 0x6a568e0, 0xc42210be60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422d87000, 0x6a551e0, 0xc4241ccd00, 0x6a568e0, 0xc42210be60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422d87000, 0x6a551e0, 0xc423228f00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc423228f00, 0x0, 0x0, 0xc420e9900d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423200550, 0xc4204a4700, 0x6a551e0, 0xc423228f00, 0xc42117dc00, 0x5400, 0x15000, 0xc420724a90, 0xc422f7a960, 0xc423200540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114311 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4211ebb80, 0x45563f3, 0xc4243c91e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4211ebb80, 0x5d2c558, 0xc4243c9420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420c0ede0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420c0ede0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc424a42c78, 0x5b6ec40, 0xc422b69bc0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424a5bac0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc424a42c78, 0x5b6ec40, 0xc422b69bc0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc424a5bac0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc424a42c78, 0x5b6ec40, 0xc422b69bc0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424a5bac0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc424a42c78, 0x5b6ec40, 0xc422b69bc0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241cafa8, 0x6a551e0, 0xc424a5bac0, 0xc424a42c78, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7f936904a) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421037500, 0xc424a42c60, 0x6a551e0, 0xc424a5bac0, 0xc400000008, 0xc424a17b80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421037500, 0x6a551e0, 0xc424a5bac0, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421037500, 0x6a551e0, 0xc424a5bac0, 0xc424a42bb0, 0xc422b69aa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115230 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423d65470, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d65470, 0x15179d5721b728d8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412ec00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d65470, 0x15179d5721b728d8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412ec00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423d65470, 0x15179d5721b728d8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412ec00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423d65440, 0x15179d5721b728d8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412ec00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423d653e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412ec00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423d653e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423d653b0, 0x5c4d11c, 0x10, 0xc422dc92c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423d653b0, 0xc420343f80, 0xc423d653b0, 0xc4267f9568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423d653b0, 0xc420343f80, 0xc420b53500, 0xc423f805f8, 0xc423a460c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423d65260, 0x5c13340, 0xc420343f80, 0x5c13340, 0xc420343f80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423d65260, 0x5c13340, 0xc420343f80, 0xc420eabf60, 0xc420eabf80, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423d65260, 0x5c13340, 0xc420343f80, 0xc420eabf60, 0xc420eabf80, 0x6, 0xc4267f9968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423d65260, 0xc420343f10, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424925c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420de2140, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420de2140, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420de2140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 1886 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc4204e0840, 0xc4202d8b90, 0x7, 0xc42269cbe0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc4204e0840, 0x7, 0xc42269cbe0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc4204e0840, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc4204e0840, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc4204e0840, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc4204e0840, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42117d180, 0x6a552a0, 0xc4204e07e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc423ef8c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4210d9860, 0x6a551e0, 0xc423ef8c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc423ef8c40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc423ef8c40, 0xc42356f878, 0xc42546b500, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc4210d9860, 0x6a551e0, 0xc423ef8c40, 0xc42546b500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4222783d0, 0x6a551e0, 0xc423ef8c40, 0xc4229e0a20, 0x13, 0x20, 0xcb4, 0x6a651a0, 0xc421bc0c00, 0xc422a42c20, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc422557140, 0x6a551e0, 0xc423ef8c40, 0xc422278440, 0xc4222783d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc422557140, 0xc4223476a8, 0xc423570b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc423ef9040, 0x5c46757, 0xc, 0x6a4c120, 0xc422557140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc423ef9040, 0x6a4c120, 0xc422557140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc423ef8c40, 0x6a4c120, 0xc422557140, 0x5d28ea0, 0x0, 0x0, 0xc422557170, 0xc422eb2b60, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc423ef8c40, 0xc422278440, 0xc4222783d0, 0x6a4c120, 0xc422557140, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc423ef8c40, 0xc422278440, 0xc4222783d0, 0x6a4c120, 0xc422557140, 0x0, 0xc422eb2b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422278648, 0x6a551e0, 0xc423ef8c40, 0xc422278440, 0xc4222783d0, 0xa491284112cc296d, 0xc423571590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422278000, 0x6a551e0, 0xc423ef8c40, 0xc4222783d0, 0x1, 0x73e91b0, 0xc423eebc20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422278000, 0x6a551e0, 0xc423ef8c40, 0x6a568e0, 0xc423878a20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422278000, 0x6a551e0, 0xc423ef8c40, 0x6a568e0, 0xc423878a20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422278000, 0x6a551e0, 0xc423ef8c40, 0x6a568e0, 0xc423878a20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422278000, 0x6a551e0, 0xc420b4d200, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc420b4d200, 0x0, 0x0, 0xc422c3504a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc42221c800, 0xc4204a4700, 0x6a551e0, 0xc420b4d200, 0xc42117cfc0, 0x5400, 0x15000, 0xc420724a90, 0xc42168f3d0, 0xc42221c7f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1887 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc420c0ed80, 0xc4202d8b90, 0x7, 0xc42269d810, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc420c0ed80, 0x7, 0xc42269d810, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc420c0ed80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42120df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc420c0ed80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42120df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc420c0ed80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42120df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc420c0ed80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42120de00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4247c5500, 0x6a552a0, 0xc420c0ec90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42120de00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc422e30240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42120de00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423867380, 0x6a551e0, 0xc422e30240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc422e30240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc422e30240, 0xc4245f5878, 0xc423ebb500, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc423867380, 0x6a551e0, 0xc422e30240, 0xc423ebb500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4222793d0, 0x6a551e0, 0xc422e30240, 0xc420355300, 0x13, 0x20, 0xcc9, 0x6a651a0, 0xc420b52600, 0xc424741ab0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4219f6690, 0x6a551e0, 0xc422e30240, 0xc422279440, 0xc4222793d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4219f6690, 0xc422347828, 0xc4245f6b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42397c880, 0x5c46757, 0xc, 0x6a4c120, 0xc4219f6690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42397c880, 0x6a4c120, 0xc4219f6690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc422e30240, 0x6a4c120, 0xc4219f6690, 0x5d28ea0, 0x0, 0x0, 0xc4219f66c0, 0xc421c22860, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc422e30240, 0xc422279440, 0xc4222793d0, 0x6a4c120, 0xc4219f6690, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc422e30240, 0xc422279440, 0xc4222793d0, 0x6a4c120, 0xc4219f6690, 0x0, 0xc421c22860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422279648, 0x6a551e0, 0xc422e30240, 0xc422279440, 0xc4222793d0, 0x12edfd64878253d8, 0xc4245f7590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422279000, 0x6a551e0, 0xc422e30240, 0xc4222793d0, 0x1, 0x73e91b0, 0xc42273a7e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422279000, 0x6a551e0, 0xc422e30240, 0x6a568e0, 0xc423800c60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422279000, 0x6a551e0, 0xc422e30240, 0x6a568e0, 0xc423800c60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422279000, 0x6a551e0, 0xc422e30240, 0x6a568e0, 0xc423800c60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422279000, 0x6a551e0, 0xc42104b840, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc42104b840, 0x0, 0x0, 0xc42278c035, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4215ed2c0, 0xc4204a4700, 0x6a551e0, 0xc42104b840, 0xc42117d340, 0x5400, 0x15000, 0xc420724a90, 0xc4225926d0, 0xc4215ed2b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 2039 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc423d64780, 0xc420518e60, 0x7, 0xc420e815e0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc423d64780, 0x7, 0xc420e815e0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc423d64780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412e600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc423d64780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412e600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc423d64780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412e600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc423d64780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412e500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423644700, 0x6a552a0, 0xc423d64720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412e500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc42403afc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412e500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422bdbba0, 0x6a551e0, 0xc42403afc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc42403afc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc42403afc0, 0xc42400d878, 0xc423ea7500, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422bdbba0, 0x6a551e0, 0xc42403afc0, 0xc423ea7500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422d423d0, 0x6a551e0, 0xc42403afc0, 0xc422682f40, 0x13, 0x20, 0xcd2, 0x6a651a0, 0xc420b52300, 0xc4246074a0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4211179e0, 0x6a551e0, 0xc42403afc0, 0xc422d42440, 0xc422d423d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4211179e0, 0xc422c63228, 0xc42400eb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc424105680, 0x5c46757, 0xc, 0x6a4c120, 0xc4211179e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc424105680, 0x6a4c120, 0xc4211179e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc42403afc0, 0x6a4c120, 0xc4211179e0, 0x5d28ea0, 0x0, 0x0, 0xc421117a10, 0xc420eaa900, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc42403afc0, 0xc422d42440, 0xc422d423d0, 0x6a4c120, 0xc4211179e0, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc42403afc0, 0xc422d42440, 0xc422d423d0, 0x6a4c120, 0xc4211179e0, 0x0, 0xc420eaa900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422d42648, 0x6a551e0, 0xc42403afc0, 0xc422d42440, 0xc422d423d0, 0x3794c3540f8a7c17, 0xc42400f590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422d42000, 0x6a551e0, 0xc42403afc0, 0xc422d423d0, 0x1, 0x73e91b0, 0xc421875560, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422d42000, 0x6a551e0, 0xc42403afc0, 0x6a568e0, 0xc422b68600, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422d42000, 0x6a551e0, 0xc42403afc0, 0x6a568e0, 0xc422b68600, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422d42000, 0x6a551e0, 0xc42403afc0, 0x6a568e0, 0xc422b68600, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422d42000, 0x6a551e0, 0xc423229940, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc423229940, 0x0, 0x0, 0xc421fc400d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4232009a0, 0xc42100a000, 0x6a551e0, 0xc423229940, 0xc422fc9340, 0x5400, 0x15000, 0xc42136a190, 0xc422f7ad50, 0xc423200990) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1840 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc42248ec60, 0xc4202d8b90, 0x7, 0xc422791540, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc42248ec60, 0x7, 0xc422791540, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc42248ec60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4238a2800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc42248ec60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4238a2800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc42248ec60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4238a2800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc42248ec60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4238a2700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e6ba40, 0x6a552a0, 0xc42248ebd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4238a2700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc424a5bec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4238a2700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc421b2cb60, 0x6a551e0, 0xc424a5bec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc421b2cb60, 0x6a551e0, 0xc424a5bec0, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc421b2cb60, 0x6a551e0, 0xc424a5bec0, 0xc421b2cb60, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc421b2cb60, 0x6a551e0, 0xc424a5bec0, 0xc423f3dbb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc421b6f000, 0x6a551e0, 0xc424a5bec0, 0x6a568e0, 0xc422b68de0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421b6f000, 0x6a551e0, 0xc424a5bec0, 0x6a568e0, 0xc422b68de0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421b6f000, 0x6a551e0, 0xc4212f0d80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc4212f0d80, 0x0, 0x0, 0xc421e9400d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4228741f0, 0xc4204a4700, 0x6a551e0, 0xc4212f0d80, 0xc421a801c0, 0x5400, 0x15000, 0xc420724a90, 0xc4226eff70, 0xc4228741e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114498 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124ca50, 0x15179d5712895750, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124ca50, 0x15179d5712895750, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42124ca50, 0x15179d5712895750, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42124c9c0, 0x15179d5712895750, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42124c870, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42124c870, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42124c870, 0x5c4d11c, 0x10, 0xc4237199b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42124c870, 0xc422c75c48, 0xc42124c870, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42124c870, 0xc422c75c48, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f58fc0, 0xc422c75c30, 0x6a551e0, 0xc4241d3080, 0xc400000008, 0xc424918340, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f58fc0, 0x6a551e0, 0xc4241d3080, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f58fc0, 0x6a551e0, 0xc4241d3080, 0xc422c75b80, 0xc42168c720) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 1888 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc421649dd0, 0xc420518e60, 0x7, 0xc4239c3040, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc421649dd0, 0x7, 0xc4239c3040, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc421649dd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6bb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc421649dd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6bb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc421649dd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6bb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc421649dd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6ba00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42435e700, 0x6a552a0, 0xc421649d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6ba00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc423315900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6ba00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423c769c0, 0x6a551e0, 0xc423315900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc423315900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc423315900, 0xc42264b878, 0xc422a07500, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc423c769c0, 0x6a551e0, 0xc423315900, 0xc422a07500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422ee83d0, 0x6a551e0, 0xc423315900, 0xc422682ec0, 0x13, 0x20, 0xcc0, 0x6a651a0, 0xc421baaf00, 0xc4249caea0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc421648bd0, 0x6a551e0, 0xc423315900, 0xc422ee8440, 0xc422ee83d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc421648bd0, 0xc422347e28, 0xc42264cb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc4220a8d00, 0x5c46757, 0xc, 0x6a4c120, 0xc421648bd0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc4220a8d00, 0x6a4c120, 0xc421648bd0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc423315900, 0x6a4c120, 0xc421648bd0, 0x5d28ea0, 0x0, 0x0, 0xc421648c00, 0xc422432ea0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc423315900, 0xc422ee8440, 0xc422ee83d0, 0x6a4c120, 0xc421648bd0, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc423315900, 0xc422ee8440, 0xc422ee83d0, 0x6a4c120, 0xc421648bd0, 0x0, 0xc422432ea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422ee8648, 0x6a551e0, 0xc423315900, 0xc422ee8440, 0xc422ee83d0, 0x435a713324482aae, 0xc42264d590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422ee8000, 0x6a551e0, 0xc423315900, 0xc422ee83d0, 0x1, 0x73e91b0, 0xc4219c15f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422ee8000, 0x6a551e0, 0xc423315900, 0x6a568e0, 0xc423a091a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422ee8000, 0x6a551e0, 0xc423315900, 0x6a568e0, 0xc423a091a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422ee8000, 0x6a551e0, 0xc423315900, 0x6a568e0, 0xc423a091a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422ee8000, 0x6a551e0, 0xc42104bbc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc42104bbc0, 0x0, 0x0, 0xc421c2e04a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4215edb30, 0xc42100a000, 0x6a551e0, 0xc42104bbc0, 0xc42117d6c0, 0x5400, 0x15000, 0xc42136a190, 0xc422593090, 0xc4215edb20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115240 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42124db30, 0xc42311ca80, 0xc423e0bec0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124db30, 0x15179d56fe4cefb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124db30, 0x15179d56fe4cefb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42124db30, 0x15179d56fe4cefb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42124db00, 0x15179d56fe4cefb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42124daa0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42124daa0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42124da70, 0x5c4d11c, 0x10, 0xc4243c92c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42124da70, 0xc42311ca10, 0xc42124da70, 0xc425463568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42124da70, 0xc42311ca10, 0xc420b53500, 0xc4241ca5f8, 0xc422495600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4230092c0, 0x5c13340, 0xc42311ca10, 0x5c13340, 0xc42311ca10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4230092c0, 0x5c13340, 0xc42311ca10, 0xc422e27920, 0xc422e27940, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4230092c0, 0x5c13340, 0xc42311ca10, 0xc422e27920, 0xc422e27940, 0x6, 0xc425463968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4230092c0, 0xc42311c9a0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0x5b8e380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8640, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8640, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8640) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 10901 [IO wait]: internal/poll.runtime_pollWait(0x77246c0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421c01198, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421c01198, 0xc421286000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421c01180, 0xc421286000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421c01180, 0xc421286000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc423f80ec8, 0xc421286000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc422313220, 0xc421286000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4225b5f80, 0x73ccbf8, 0xc422313220, 0x5, 0xc422313220, 0xc2) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42232f180, 0x5d2d117, 0xc42232f2a0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42232f180, 0xc423ac8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421f75f40, 0xc423ac8000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc42209b8e0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc42209b8e0, 0xbe9de8a3295a6d00, 0x7c9b3bd5f, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc42209ba10, 0x6a46020, 0xc42209b8e0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc42209b880, 0x6a551e0, 0xc4229b7180, 0xc422b769f0, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc421154640, 0x6a611e0, 0xc42232f180, 0x0, 0x0, 0xc422c1a00d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc421154640, 0x6a611e0, 0xc42232f180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4225b5f20, 0x6a61a80, 0xc422313220, 0xc4212cb5f0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc422313220) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc422313220, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 1841 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42124c780, 0xc4206a4230, 0x7, 0xc4237bdd10, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42124c780, 0x7, 0xc4237bdd10, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42124c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6b700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42124c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6b700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42124c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6b700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42124c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6b600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4245af6c0, 0x6a552a0, 0xc42124c720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6b600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc422494b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f6b600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422343a00, 0x6a551e0, 0xc422494b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc422494b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc422494b00, 0xc4243ed878, 0xc422c57500, 0xda, 0xda) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422343a00, 0x6a551e0, 0xc422494b00, 0xc422c57500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422f143d0, 0x6a551e0, 0xc422494b00, 0xc422490180, 0x12, 0x20, 0xcb7, 0x6a651a0, 0xc4223ac000, 0xc424344690, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc423b4f170, 0x6a551e0, 0xc422494b00, 0xc422f14440, 0xc422f143d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc423b4f170, 0xc422453e28, 0xc4243eeb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42335b9c0, 0x5c46757, 0xc, 0x6a4c120, 0xc423b4f170) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42335b9c0, 0x6a4c120, 0xc423b4f170) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc422494b00, 0x6a4c120, 0xc423b4f170, 0x5d28ea0, 0x0, 0x0, 0xc423b4f1a0, 0xc422d4c000, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc422494b00, 0xc422f14440, 0xc422f143d0, 0x6a4c120, 0xc423b4f170, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc422494b00, 0xc422f14440, 0xc422f143d0, 0x6a4c120, 0xc423b4f170, 0x0, 0xc422d4c000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422f14648, 0x6a551e0, 0xc422494b00, 0xc422f14440, 0xc422f143d0, 0xee3bb6301ee5024b, 0xc4243ef590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422f14000, 0x6a551e0, 0xc422494b00, 0xc422f143d0, 0x1, 0x73e91b0, 0xc4236930e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422f14000, 0x6a551e0, 0xc422494b00, 0x6a568e0, 0xc421238f60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422f14000, 0x6a551e0, 0xc422494b00, 0x6a568e0, 0xc421238f60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422f14000, 0x6a551e0, 0xc422494b00, 0x6a568e0, 0xc421238f60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422f14000, 0x6a551e0, 0xc4212f1a00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc4212f1a00, 0x0, 0x0, 0xc42272a020, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422875400, 0xc42100a460, 0x6a551e0, 0xc4212f1a00, 0xc421a80380, 0x5400, 0x15000, 0xc4206d5810, 0xc4224d97b0, 0xc4228753f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1938 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc42370f410, 0xc420518e60, 0x7, 0xc423dea410, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc42370f410, 0x7, 0xc423dea410, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc42370f410, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42341d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc42370f410, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42341d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc42370f410, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42341d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc42370f410, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42341d300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e06000, 0x6a552a0, 0xc42370f3b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42341d300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc423314300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42341d300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422af21a0, 0x6a551e0, 0xc423314300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc422af21a0, 0x6a551e0, 0xc423314300, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc422af21a0, 0x6a551e0, 0xc423314300, 0xc422af21a0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc422af21a0, 0x6a551e0, 0xc423314300, 0xc421a2bbb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422f15000, 0x6a551e0, 0xc423314300, 0x6a568e0, 0xc4224a53e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422f15000, 0x6a551e0, 0xc423314300, 0x6a568e0, 0xc4224a53e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422f15000, 0x6a551e0, 0xc4212f1b00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc4212f1b00, 0x0, 0x0, 0xc420cf904a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422875460, 0xc42100a000, 0x6a551e0, 0xc4212f1b00, 0xc421a80540, 0x5400, 0x15000, 0xc42136a190, 0xc4224d97c0, 0xc422875450) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1939 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4225d48d0, 0xc4206a4230, 0x7, 0xc420b7a1e0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4225d48d0, 0x7, 0xc420b7a1e0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4225d48d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4225d48d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4225d48d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4225d48d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4224d5880, 0x6a552a0, 0xc4225d4840, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4231b6f40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422342680, 0x6a551e0, 0xc4231b6f40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4231b6f40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4231b6f40, 0xc423281878, 0xc42532b500, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422342680, 0x6a551e0, 0xc4231b6f40, 0xc42532b500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422f243d0, 0x6a551e0, 0xc4231b6f40, 0xc4229e1980, 0x13, 0x20, 0xcb2, 0x6a651a0, 0xc423d4c000, 0xc4231bad80, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4223d5290, 0x6a551e0, 0xc4231b6f40, 0xc422f24440, 0xc422f243d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4223d5290, 0xc422f164a8, 0xc423282b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42141c240, 0x5c46757, 0xc, 0x6a4c120, 0xc4223d5290) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42141c240, 0x6a4c120, 0xc4223d5290) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4231b6f40, 0x6a4c120, 0xc4223d5290, 0x5d28ea0, 0x0, 0x0, 0xc4223d52c0, 0xc420eab4c0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4231b6f40, 0xc422f24440, 0xc422f243d0, 0x6a4c120, 0xc4223d5290, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4231b6f40, 0xc422f24440, 0xc422f243d0, 0x6a4c120, 0xc4223d5290, 0x0, 0xc420eab4c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422f24648, 0x6a551e0, 0xc4231b6f40, 0xc422f24440, 0xc422f243d0, 0x170411770d74731e, 0xc423283590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422f24000, 0x6a551e0, 0xc4231b6f40, 0xc422f243d0, 0x1, 0x73e91b0, 0xc424a6f710, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422f24000, 0x6a551e0, 0xc4231b6f40, 0x6a568e0, 0xc421a1c420, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422f24000, 0x6a551e0, 0xc4231b6f40, 0x6a568e0, 0xc421a1c420, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422f24000, 0x6a551e0, 0xc4231b6f40, 0x6a568e0, 0xc421a1c420, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422f24000, 0x6a551e0, 0xc4212f1bc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc4212f1bc0, 0x0, 0x0, 0xc422480022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4228754c0, 0xc42100a460, 0x6a551e0, 0xc4212f1bc0, 0xc421a80700, 0x5400, 0x15000, 0xc4206d5810, 0xc4224d97d0, 0xc4228754b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1923 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc4225f83f0, 0xc4202d8b90, 0x7, 0xc4226e3f90, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc4225f83f0, 0x7, 0xc4226e3f90, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc4225f83f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423757600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc4225f83f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423757600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc4225f83f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423757600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc4225f83f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423757500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42435e8c0, 0x6a552a0, 0xc4225f8390, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423757500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc424918040, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423757500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423c776c0, 0x6a551e0, 0xc424918040, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc424918040, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc424918040, 0xc4237f5878, 0xc422a08a00, 0xda, 0xda) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc423c776c0, 0x6a551e0, 0xc424918040, 0xc422a08a00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422f013d0, 0x6a551e0, 0xc424918040, 0xc420016900, 0x12, 0x20, 0xcc1, 0x6a651a0, 0xc421bab800, 0xc4249cb810, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc420f2e6c0, 0x6a551e0, 0xc424918040, 0xc422f01440, 0xc422f013d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc420f2e6c0, 0xc422eec7a8, 0xc4237f6b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc421451bc0, 0x5c46757, 0xc, 0x6a4c120, 0xc420f2e6c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc421451bc0, 0x6a4c120, 0xc420f2e6c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc424918040, 0x6a4c120, 0xc420f2e6c0, 0x5d28ea0, 0x0, 0x0, 0xc420f2e6f0, 0xc420371d00, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc424918040, 0xc422f01440, 0xc422f013d0, 0x6a4c120, 0xc420f2e6c0, 0x735a000, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc424918040, 0xc422f01440, 0xc422f013d0, 0x6a4c120, 0xc420f2e6c0, 0x0, 0xc420371d00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422f01648, 0x6a551e0, 0xc424918040, 0xc422f01440, 0xc422f013d0, 0x16606f3a77e0a3f1, 0xc4237f7590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422f01000, 0x6a551e0, 0xc424918040, 0xc422f013d0, 0x1, 0x73e91b0, 0xc420f74990, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422f01000, 0x6a551e0, 0xc424918040, 0x6a568e0, 0xc420efa4e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422f01000, 0x6a551e0, 0xc424918040, 0x6a568e0, 0xc420efa4e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422f01000, 0x6a551e0, 0xc424918040, 0x6a568e0, 0xc420efa4e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422f01000, 0x6a551e0, 0xc42132a000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc42132a000, 0x0, 0x0, 0xc42278d022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4215edfa0, 0xc4204a4700, 0x6a551e0, 0xc42132a000, 0xc42117ddc0, 0x5400, 0x15000, 0xc420724a90, 0xc4225935a0, 0xc4215edf90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1969 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).watcher(0xc4228d1290) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:110 +0x1ac created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccResolverWrapper).start /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/resolver_conn_wrapper.go:96 +0x3f goroutine 1891 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc423b4e420, 0xc4202d8b90, 0x7, 0xc420bd28c0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc423b4e420, 0x7, 0xc420bd28c0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc423b4e420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440cc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc423b4e420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440cc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc423b4e420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440cc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc423b4e420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440ca00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4224d48c0, 0x6a552a0, 0xc423b4e3c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440ca00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc424a75400, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440ca00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4210d89c0, 0x6a551e0, 0xc424a75400, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc4210d89c0, 0x6a551e0, 0xc424a75400, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc4210d89c0, 0x6a551e0, 0xc424a75400, 0xc4210d89c0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc4210d89c0, 0x6a551e0, 0xc424a75400, 0xc422f5dbb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc4229db000, 0x6a551e0, 0xc424a75400, 0x6a568e0, 0xc423340180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4229db000, 0x6a551e0, 0xc424a75400, 0x6a568e0, 0xc423340180, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4229db000, 0x6a551e0, 0xc42101ba00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc42101ba00, 0x0, 0x0, 0xc420e9b00d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4228672b0, 0xc4204a4700, 0x6a551e0, 0xc42101ba00, 0xc422fc8000, 0x5400, 0x15000, 0xc420724a90, 0xc422bea5b0, 0xc4228672a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1965 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc420c0ed50, 0xc4202d8b90, 0x7, 0xc4237bc7d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc420c0ed50, 0x7, 0xc4237bc7d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc420c0ed50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440c600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc420c0ed50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440c600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc420c0ed50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42440c600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc420c0ed50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412fb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e076c0, 0x6a552a0, 0xc420c0ecf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412fb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc422e30180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42412fb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422342000, 0x6a551e0, 0xc422e30180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc422342000, 0x6a551e0, 0xc422e30180, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc422342000, 0x6a551e0, 0xc422e30180, 0xc422342000, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc422342000, 0x6a551e0, 0xc422e30180, 0xc423333bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc423114000, 0x6a551e0, 0xc422e30180, 0x6a568e0, 0xc4222272c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc423114000, 0x6a551e0, 0xc422e30180, 0x6a568e0, 0xc4222272c0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc423114000, 0x6a551e0, 0xc422bd81c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc422bd81c0, 0x0, 0x0, 0xc422520020, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422e81190, 0xc4204a4700, 0x6a551e0, 0xc422bd81c0, 0xc4224d4e00, 0x5400, 0x15000, 0xc420724a90, 0xc422f7a7a0, 0xc422e81180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 2208 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4224276e0, 0xc4206a4230, 0x7, 0xc4223aad70, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4224276e0, 0x7, 0xc4223aad70, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4224276e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4224276e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4224276e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4224276e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42435f880, 0x6a552a0, 0xc422427680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4241cd200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4217e2600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc421c3e680, 0x6a551e0, 0xc4241cd200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4241cd200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4241cd200, 0xc423457878, 0xc424b91800, 0xda, 0xda) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc421c3e680, 0x6a551e0, 0xc4241cd200, 0xc424b91800, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4232fa3d0, 0x6a551e0, 0xc4241cd200, 0xc4226822e0, 0x12, 0x20, 0xcd1, 0x6a651a0, 0xc426460300, 0xc423678fa0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4239f9ef0, 0x6a551e0, 0xc4241cd200, 0xc4232fa440, 0xc4232fa3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4239f9ef0, 0xc422f173a8, 0xc423458b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc4226c5b00, 0x5c46757, 0xc, 0x6a4c120, 0xc4239f9ef0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc4226c5b00, 0x6a4c120, 0xc4239f9ef0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4241cd200, 0x6a4c120, 0xc4239f9ef0, 0x5d28ea0, 0x0, 0x0, 0xc4239f9f20, 0xc4203834e0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4241cd200, 0xc4232fa440, 0xc4232fa3d0, 0x6a4c120, 0xc4239f9ef0, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4241cd200, 0xc4232fa440, 0xc4232fa3d0, 0x6a4c120, 0xc4239f9ef0, 0x0, 0xc4203834e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc4232fa648, 0x6a551e0, 0xc4241cd200, 0xc4232fa440, 0xc4232fa3d0, 0x71078d8758f77076, 0xc423459590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc4232fa000, 0x6a551e0, 0xc4241cd200, 0xc4232fa3d0, 0x1, 0x73e91b0, 0xc4228c0000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc4232fa000, 0x6a551e0, 0xc4241cd200, 0x6a568e0, 0xc42353cfc0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc4232fa000, 0x6a551e0, 0xc4241cd200, 0x6a568e0, 0xc42353cfc0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4232fa000, 0x6a551e0, 0xc4241cd200, 0x6a568e0, 0xc42353cfc0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4232fa000, 0x6a551e0, 0xc4229b6240, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc4229b6240, 0x0, 0x0, 0xc42294000d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc420378720, 0xc42100a460, 0x6a551e0, 0xc4229b6240, 0xc42293a000, 0x5400, 0x15000, 0xc4206d5810, 0xc422592260, 0xc420378700) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114610 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc42359dc20, 0xc420518e60, 0x7, 0xc423c32140, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc42359dc20, 0x7, 0xc423c32140, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc42359dc20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc42359dc20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc42359dc20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc42359dc20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc421c3ba40, 0x6a552a0, 0xc423b45e30, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc421c3ba40, 0x6a552a0, 0xc423b45e30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc423b45770) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc423b45770, 0xc420cbdb00, 0x29, 0x6a652e0, 0xc420358700, 0xc4211068f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 1929 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420c5a330, 0xc4206a4230, 0x7, 0xc4224f1040, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420c5a330, 0x7, 0xc4224f1040, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420c5a330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421314700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420c5a330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421314700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420c5a330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421314700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420c5a330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421314400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ee4380, 0x6a552a0, 0xc420c5a2d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421314400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4231b7dc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421314400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422bf24e0, 0x6a551e0, 0xc4231b7dc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4231b7dc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4231b7dc0, 0xc4232f5878, 0xc4250f3c00, 0xda, 0xda) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422bf24e0, 0x6a551e0, 0xc4231b7dc0, 0xc4250f3c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc42307c3d0, 0x6a551e0, 0xc4231b7dc0, 0xc422490fc0, 0x12, 0x20, 0xcba, 0x6a651a0, 0xc4211de300, 0xc42437f0c0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc422ed3b30, 0x6a551e0, 0xc4231b7dc0, 0xc42307c440, 0xc42307c3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc422ed3b30, 0xc422eecf28, 0xc4232f6b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc4226c4980, 0x5c46757, 0xc, 0x6a4c120, 0xc422ed3b30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc4226c4980, 0x6a4c120, 0xc422ed3b30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4231b7dc0, 0x6a4c120, 0xc422ed3b30, 0x5d28ea0, 0x0, 0x0, 0xc422ed3b60, 0xc421ebf4a0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4231b7dc0, 0xc42307c440, 0xc42307c3d0, 0x6a4c120, 0xc422ed3b30, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4231b7dc0, 0xc42307c440, 0xc42307c3d0, 0x6a4c120, 0xc422ed3b30, 0x0, 0xc421ebf4a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc42307c648, 0x6a551e0, 0xc4231b7dc0, 0xc42307c440, 0xc42307c3d0, 0x8354785b403396c6, 0xc4232f7590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc42307c000, 0x6a551e0, 0xc4231b7dc0, 0xc42307c3d0, 0x1, 0x73e91b0, 0xc4241d1b90, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc42307c000, 0x6a551e0, 0xc4231b7dc0, 0x6a568e0, 0xc422226660, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc42307c000, 0x6a551e0, 0xc4231b7dc0, 0x6a568e0, 0xc422226660, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc42307c000, 0x6a551e0, 0xc4231b7dc0, 0x6a568e0, 0xc422226660, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc42307c000, 0x6a551e0, 0xc42132b140, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc42132b140, 0x0, 0x0, 0xc4224ca022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422f415e0, 0xc42100a460, 0x6a551e0, 0xc42132b140, 0xc422fd21c0, 0x5400, 0x15000, 0xc4206d5810, 0xc422bde1e0, 0xc422f415d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1892 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc4211165d0, 0xc420518e60, 0x7, 0xc420e81f40, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc4211165d0, 0x7, 0xc420e81f40, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc4211165d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421206c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc4211165d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421206c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc4211165d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421206c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc4211165d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421206b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e09180, 0x6a552a0, 0xc421116510, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421206b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc42160f080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421206b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc42386ed00, 0x6a551e0, 0xc42160f080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc42160f080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc42160f080, 0xc422f75878, 0xc42235d100, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc42386ed00, 0x6a551e0, 0xc42160f080, 0xc42235d100, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4230783d0, 0x6a551e0, 0xc42160f080, 0xc4229e12e0, 0x13, 0x20, 0xcb5, 0x6a651a0, 0xc426461500, 0xc424607e60, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc42166a360, 0x6a551e0, 0xc42160f080, 0xc423078440, 0xc4230783d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc42166a360, 0xc422c639a8, 0xc422f76b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc423ef9d00, 0x5c46757, 0xc, 0x6a4c120, 0xc42166a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc423ef9d00, 0x6a4c120, 0xc42166a360) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc42160f080, 0x6a4c120, 0xc42166a360, 0x5d28ea0, 0x0, 0x0, 0xc42166a390, 0xc422eb3aa0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc42160f080, 0xc423078440, 0xc4230783d0, 0x6a4c120, 0xc42166a360, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc42160f080, 0xc423078440, 0xc4230783d0, 0x6a4c120, 0xc42166a360, 0x0, 0xc422eb3aa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc423078648, 0x6a551e0, 0xc42160f080, 0xc423078440, 0xc4230783d0, 0x4b8aae0d8259c512, 0xc422f77590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc423078000, 0x6a551e0, 0xc42160f080, 0xc4230783d0, 0x1, 0x73e91b0, 0xc423be57a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc423078000, 0x6a551e0, 0xc42160f080, 0x6a568e0, 0xc420e87c80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc423078000, 0x6a551e0, 0xc42160f080, 0x6a568e0, 0xc420e87c80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc423078000, 0x6a551e0, 0xc42160f080, 0x6a568e0, 0xc420e87c80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc423078000, 0x6a551e0, 0xc42101bec0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc42101bec0, 0x0, 0x0, 0xc421f0204a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422867b00, 0xc42100a000, 0x6a551e0, 0xc42101bec0, 0xc422fc8380, 0x5400, 0x15000, 0xc42136a190, 0xc422beb170, 0xc422867af0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115165 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc421180bd0, 0xc420dd33b0, 0xc423c5d020, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421180bd0, 0x15179d5722b6c658, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421180bd0, 0x15179d5722b6c658, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421180bd0, 0x15179d5722b6c658, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421180a50, 0x15179d5722b6c658, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4211809f0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4211809f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4211809f0, 0x5c4d11c, 0x10, 0xc423c3d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4211809f0, 0xc422a79408, 0xc4211809f0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4211809f0, 0xc422a79408, 0x1f, 0xc420896630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420dd3340, 0xc422a793f0, 0x6a551e0, 0xc42456ee40, 0xc400000008, 0xc421451500, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420dd3340, 0x6a551e0, 0xc42456ee40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420dd3340, 0x6a551e0, 0xc42456ee40, 0xc422a79340, 0xc423c5cd80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 2046 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4216854a0, 0xc4206a4230, 0x7, 0xc42382d770, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4216854a0, 0x7, 0xc42382d770, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4216854a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4216854a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4216854a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4216854a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2ddc0, 0x6a552a0, 0xc421685350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4231b7680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422089100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc420c3e340, 0x6a551e0, 0xc4231b7680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc420c3e340, 0x6a551e0, 0xc4231b7680, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc420c3e340, 0x6a551e0, 0xc4231b7680, 0xc420c3e340, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc420c3e340, 0x6a551e0, 0xc4231b7680, 0xc4245bdbb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422d43000, 0x6a551e0, 0xc4231b7680, 0x6a568e0, 0xc423141d40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422d43000, 0x6a551e0, 0xc4231b7680, 0x6a568e0, 0xc423141d40, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422d43000, 0x6a551e0, 0xc42121b380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc42121b380, 0x0, 0x0, 0xc42248100d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423200de0, 0xc42100a460, 0x6a551e0, 0xc42121b380, 0xc422fc9c00, 0x5400, 0x15000, 0xc4206d5810, 0xc422f7b8e0, 0xc423200dd0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1950 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423710db0, 0xc4206a4230, 0x7, 0xc423729090, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423710db0, 0x7, 0xc423729090, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423710db0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42272c200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423710db0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42272c200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423710db0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42272c200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423710db0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42272c100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4249ce700, 0x6a552a0, 0xc423710c90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42272c100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc423aa1bc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42272c100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423614680, 0x6a551e0, 0xc423aa1bc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc423aa1bc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc423aa1bc0, 0xc4232db878, 0xc4219a8a00, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc423614680, 0x6a551e0, 0xc423aa1bc0, 0xc4219a8a00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4231153d0, 0x6a551e0, 0xc423aa1bc0, 0xc424b93120, 0x13, 0x20, 0xcc8, 0x6a651a0, 0xc4211de600, 0xc4249cb820, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc423f5c7b0, 0x6a551e0, 0xc423aa1bc0, 0xc423115440, 0xc4231153d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc423f5c7b0, 0xc422f179a8, 0xc4232dcb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc423910400, 0x5c46757, 0xc, 0x6a4c120, 0xc423f5c7b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc423910400, 0x6a4c120, 0xc423f5c7b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc423aa1bc0, 0x6a4c120, 0xc423f5c7b0, 0x5d28ea0, 0x0, 0x0, 0xc423f5c7e0, 0xc42230b960, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc423aa1bc0, 0xc423115440, 0xc4231153d0, 0x6a4c120, 0xc423f5c7b0, 0x735a000, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc423aa1bc0, 0xc423115440, 0xc4231153d0, 0x6a4c120, 0xc423f5c7b0, 0x0, 0xc42230b960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc423115648, 0x6a551e0, 0xc423aa1bc0, 0xc423115440, 0xc4231153d0, 0xc3bd2b925b58721e, 0xc4232dd590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc423115000, 0x6a551e0, 0xc423aa1bc0, 0xc4231153d0, 0x1, 0x73e91b0, 0xc420c038c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc423115000, 0x6a551e0, 0xc423aa1bc0, 0x6a568e0, 0xc4236a43c0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc423115000, 0x6a551e0, 0xc423aa1bc0, 0x6a568e0, 0xc4236a43c0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc423115000, 0x6a551e0, 0xc423aa1bc0, 0x6a568e0, 0xc4236a43c0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc423115000, 0x6a551e0, 0xc422f9e780, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc422f9e780, 0x0, 0x0, 0xc42272604a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423006300, 0xc42100a460, 0x6a551e0, 0xc422f9e780, 0xc421a80c40, 0x5400, 0x15000, 0xc4206d5810, 0xc421b3f9f0, 0xc4230062f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 2038 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc4224f3140, 0xc420518e60, 0x7, 0xc4243e1630, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc4224f3140, 0x7, 0xc4243e1630, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc4224f3140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc4224f3140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc4224f3140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc4224f3140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4249cf6c0, 0x6a552a0, 0xc4224f30e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc424732040, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc42282a9c0, 0x6a551e0, 0xc424732040, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc42282a9c0, 0x6a551e0, 0xc424732040, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc42282a9c0, 0x6a551e0, 0xc424732040, 0xc42282a9c0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc42282a9c0, 0x6a551e0, 0xc424732040, 0xc4230f5bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc421b16000, 0x6a551e0, 0xc424732040, 0x6a568e0, 0xc42193f140, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421b16000, 0x6a551e0, 0xc424732040, 0x6a568e0, 0xc42193f140, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421b16000, 0x6a551e0, 0xc423229880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc423229880, 0x0, 0x0, 0xc4222c104a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423200950, 0xc42100a000, 0x6a551e0, 0xc423229880, 0xc422fc9180, 0x5400, 0x15000, 0xc42136a190, 0xc422f7ad40, 0xc423200940) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 113447 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4213a5380, 0x15179d56f71afca8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4213a5380, 0x15179d56f71afca8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4213a5380, 0x15179d56f71afca8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4213a5350, 0x15179d56f71afca8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4213a52f0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4213a52f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4213a52f0, 0x5c4d11c, 0x10, 0xc4220b19b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4213a52f0, 0xc4231d3da8, 0xc4213a52f0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4213a52f0, 0xc4231d3da8, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5dd50, 0xc4231d3d90, 0x6a551e0, 0xc4231b7580, 0xc400000008, 0xc4226c4d40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5dd50, 0x6a551e0, 0xc4231b7580, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5dd50, 0x6a551e0, 0xc4231b7580, 0xc4231d3ce0, 0xc423141860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115286 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc420b26f90, 0xc4202da230, 0xc423942d20, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420b26f90, 0x15179d5708a1fc88, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421315100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420b26f90, 0x15179d5708a1fc88, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421315100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420b26f90, 0x15179d5708a1fc88, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421315100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420b26f60, 0x15179d5708a1fc88, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421315100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420b26ed0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421315100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420b26ed0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420b26ed0, 0x5c4d11c, 0x10, 0xc4231359b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420b26ed0, 0xc422a79e58, 0xc420b26ed0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420b26ed0, 0xc422a79e58, 0x1f, 0xc420bcc630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4202da150, 0xc422a79e40, 0x6a551e0, 0xc424a5b380, 0xc400000008, 0xc424a5b540, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4202da150, 0x6a551e0, 0xc424a5b380, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4202da150, 0x6a551e0, 0xc424a5b380, 0xc422a79d90, 0xc423942c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 2036 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc4218bcc60, 0xc4202d8b90, 0x7, 0xc423c403c0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc4218bcc60, 0x7, 0xc423c403c0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc4218bcc60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f01400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc4218bcc60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f01400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc4218bcc60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f01400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc4218bcc60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f01200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4245ae1c0, 0x6a552a0, 0xc4218bcc00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f01200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc4231b74c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421f01200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4232209c0, 0x6a551e0, 0xc4231b74c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4231b74c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4231b74c0, 0xc422d29878, 0xc423de0700, 0xda, 0xda) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc4232209c0, 0x6a551e0, 0xc4231b74c0, 0xc423de0700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422cb43d0, 0x6a551e0, 0xc4231b74c0, 0xc421eceb60, 0x12, 0x20, 0xccc, 0x6a651a0, 0xc424336300, 0xc42440fbe0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc422dada40, 0x6a551e0, 0xc4231b74c0, 0xc422cb4440, 0xc422cb43d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc422dada40, 0xc422eed228, 0xc422d2ab28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc424a5af40, 0x5c46757, 0xc, 0x6a4c120, 0xc422dada40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc424a5af40, 0x6a4c120, 0xc422dada40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4231b74c0, 0x6a4c120, 0xc422dada40, 0x5d28ea0, 0x0, 0x0, 0xc422dadaa0, 0xc422d4d080, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4231b74c0, 0xc422cb4440, 0xc422cb43d0, 0x6a4c120, 0xc422dada40, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4231b74c0, 0xc422cb4440, 0xc422cb43d0, 0x6a4c120, 0xc422dada40, 0x0, 0xc422d4d080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422cb4648, 0x6a551e0, 0xc4231b74c0, 0xc422cb4440, 0xc422cb43d0, 0x9eea861222dd0840, 0xc422d2b590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422cb4000, 0x6a551e0, 0xc4231b74c0, 0xc422cb43d0, 0x1, 0x73e91b0, 0xc4241585a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422cb4000, 0x6a551e0, 0xc4231b74c0, 0x6a568e0, 0xc42395dc80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422cb4000, 0x6a551e0, 0xc4231b74c0, 0x6a568e0, 0xc42395dc80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422cb4000, 0x6a551e0, 0xc4231b74c0, 0x6a568e0, 0xc42395dc80, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422cb4000, 0x6a551e0, 0xc423229500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc423229500, 0x0, 0x0, 0xc422531035, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423200740, 0xc4204a4700, 0x6a551e0, 0xc423229500, 0xc422fc8e00, 0x5400, 0x15000, 0xc420724a90, 0xc422f7aab0, 0xc423200730) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1980 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423f5c780, 0xc4206a4230, 0x7, 0xc423c33810, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423f5c780, 0x7, 0xc423c33810, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423f5c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a1400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423f5c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a1400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423f5c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a1400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423f5c780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a1300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c27dc0, 0x6a552a0, 0xc423f5c720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a1300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc42397d2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a1300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4220411e0, 0x6a551e0, 0xc42397d2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc42397d2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc42397d2c0, 0xc42442f878, 0xc42563e000, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc4220411e0, 0x6a551e0, 0xc42397d2c0, 0xc42563e000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422f253d0, 0x6a551e0, 0xc42397d2c0, 0xc421ef7060, 0x13, 0x20, 0xcbb, 0x6a651a0, 0xc42117a300, 0xc42440eda0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc421a950b0, 0x6a551e0, 0xc42397d2c0, 0xc422f25440, 0xc422f253d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc421a950b0, 0xc422eed3a8, 0xc424430b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42397da00, 0x5c46757, 0xc, 0x6a4c120, 0xc421a950b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42397da00, 0x6a4c120, 0xc421a950b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc42397d2c0, 0x6a4c120, 0xc421a950b0, 0x5d28ea0, 0x0, 0x0, 0xc421a950e0, 0xc421c239e0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc42397d2c0, 0xc422f25440, 0xc422f253d0, 0x6a4c120, 0xc421a950b0, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc42397d2c0, 0xc422f25440, 0xc422f253d0, 0x6a4c120, 0xc421a950b0, 0x0, 0xc421c239e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422f25648, 0x6a551e0, 0xc42397d2c0, 0xc422f25440, 0xc422f253d0, 0x77aff9b807246f90, 0xc424431590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422f25000, 0x6a551e0, 0xc42397d2c0, 0xc422f253d0, 0x1, 0x73e91b0, 0xc4220041b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422f25000, 0x6a551e0, 0xc42397d2c0, 0x6a568e0, 0xc4238ae960, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422f25000, 0x6a551e0, 0xc42397d2c0, 0x6a568e0, 0xc4238ae960, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422f25000, 0x6a551e0, 0xc42397d2c0, 0x6a568e0, 0xc4238ae960, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422f25000, 0x6a551e0, 0xc42302a1c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc42302a1c0, 0x0, 0x0, 0xc4216b000d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4230071c0, 0xc42100a460, 0x6a551e0, 0xc42302a1c0, 0xc421a81dc0, 0x5400, 0x15000, 0xc4206d5810, 0xc4231c36d0, 0xc4230071b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 113536 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4211174d0, 0x15179d56f8edc8a8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4211174d0, 0x15179d56f8edc8a8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4211174d0, 0x15179d56f8edc8a8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4211174a0, 0x15179d56f8edc8a8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421117410, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421117410, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421117410, 0x5c4d11c, 0x10, 0xc4241fb9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421117410, 0xc421b52908, 0xc421117410, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421117410, 0xc421b52908, 0x1f, 0xc420895630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420cea3f0, 0xc421b528f0, 0x6a551e0, 0xc42391d340, 0xc400000008, 0xc424a16080, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420cea3f0, 0x6a551e0, 0xc42391d340, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420cea3f0, 0x6a551e0, 0xc42391d340, 0xc421b52840, 0xc4244dd800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114510 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422027620, 0x15179d57133133f8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f00000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422027620, 0x15179d57133133f8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f00000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422027620, 0x15179d57133133f8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f00000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4220275f0, 0x15179d57133133f8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f00000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422027590, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f00000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422027590, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422027590, 0x5c4d11c, 0x10, 0xc4233659b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422027590, 0xc423a9ac78, 0xc422027590, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422027590, 0xc423a9ac78, 0x1f, 0xc420033630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5df80, 0xc423a9ac60, 0x6a551e0, 0xc42397ccc0, 0xc400000008, 0xc423b7ab00, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5df80, 0x6a551e0, 0xc42397ccc0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5df80, 0x6a551e0, 0xc42397ccc0, 0xc423a9aa50, 0xc4234190e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 2045 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421bf2ea0, 0xc4206a4230, 0x7, 0xc4232b0d70, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421bf2ea0, 0x7, 0xc4232b0d70, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421bf2ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422b33300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421bf2ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422b33300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421bf2ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422b33300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421bf2ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42293ba40, 0x6a552a0, 0xc421bf2e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4241cdc80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc421b13a00, 0x6a551e0, 0xc4241cdc80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4241cdc80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4241cdc80, 0xc424d3f878, 0xc425241c00, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc421b13a00, 0x6a551e0, 0xc4241cdc80, 0xc425241c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc421e243d0, 0x6a551e0, 0xc4241cdc80, 0xc421a9ce60, 0x13, 0x20, 0xcc5, 0x6a651a0, 0xc420b38f00, 0xc423679720, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc42244c510, 0x6a551e0, 0xc4241cdc80, 0xc421e24440, 0xc421e243d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc42244c510, 0xc422eed6a8, 0xc424d40b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc424918580, 0x5c46757, 0xc, 0x6a4c120, 0xc42244c510) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc424918580, 0x6a4c120, 0xc42244c510) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4241cdc80, 0x6a4c120, 0xc42244c510, 0x5d28ea0, 0x0, 0x0, 0xc42244c540, 0xc420371fa0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4241cdc80, 0xc421e24440, 0xc421e243d0, 0x6a4c120, 0xc42244c510, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4241cdc80, 0xc421e24440, 0xc421e243d0, 0x6a4c120, 0xc42244c510, 0x0, 0xc420371fa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc421e24648, 0x6a551e0, 0xc4241cdc80, 0xc421e24440, 0xc421e243d0, 0xc0bcd1f417a22609, 0xc424d41590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc421e24000, 0x6a551e0, 0xc4241cdc80, 0xc421e243d0, 0x1, 0x73e91b0, 0xc423b0ddd0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc421e24000, 0x6a551e0, 0xc4241cdc80, 0x6a568e0, 0xc42210b080, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc421e24000, 0x6a551e0, 0xc4241cdc80, 0x6a568e0, 0xc42210b080, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421e24000, 0x6a551e0, 0xc4241cdc80, 0x6a568e0, 0xc42210b080, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421e24000, 0x6a551e0, 0xc42121b200, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc42121b200, 0x0, 0x0, 0xc4216bb022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423200d90, 0xc42100a460, 0x6a551e0, 0xc42121b200, 0xc422fc9a40, 0x5400, 0x15000, 0xc4206d5810, 0xc422f7b8d0, 0xc423200d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115284 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc420d689f0, 0xc4210816c0, 0xc42395d6e0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d689f0, 0x15179d56fba9b070, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d689f0, 0x15179d56fba9b070, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420d689f0, 0x15179d56fba9b070, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420d688d0, 0x15179d56fba9b070, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420d68870, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420d68870, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420d68720, 0x5c4d11c, 0x10, 0xc424ceb2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420d68720, 0xc421081340, 0xc420d68720, 0xc4230e1568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420d68720, 0xc421081340, 0xc420b53500, 0xc421ac8420, 0xc424a5a880) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420d68360, 0x5c13340, 0xc421081340, 0x5c13340, 0xc421081340, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420d68360, 0x5c13340, 0xc421081340, 0xc422d4d820, 0xc422d4d840, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420d68360, 0x5c13340, 0xc421081340, 0xc422d4d820, 0xc422d4d840, 0x6, 0xc4230e1968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420d68360, 0xc4210812d0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc420ccfc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423643180, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423643180, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423643180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 1998 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc42302a4c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:122 +0x156 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.newCCBalancerWrapper /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:113 +0x197 goroutine 2035 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc422e59830, 0xc4202d8b90, 0x7, 0xc422fb65f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc422e59830, 0x7, 0xc422fb65f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc422e59830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc422e59830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc422e59830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc422e59830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc424838540, 0x6a552a0, 0xc422e59740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc4241cc940, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421006500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc421b12680, 0x6a551e0, 0xc4241cc940, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4241cc940, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4241cc940, 0xc4235f7878, 0xc425240700, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc421b12680, 0x6a551e0, 0xc4241cc940, 0xc425240700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc42241d3d0, 0x6a551e0, 0xc4241cc940, 0xc421070c40, 0x13, 0x20, 0xcc4, 0x6a651a0, 0xc420718c00, 0xc423678540, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4229a0000, 0x6a551e0, 0xc4241cc940, 0xc42241d440, 0xc42241d3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4229a0000, 0xc4231d41a8, 0xc4235f8b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc424105040, 0x5c46757, 0xc, 0x6a4c120, 0xc4229a0000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc424105040, 0x6a4c120, 0xc4229a0000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4241cc940, 0x6a4c120, 0xc4229a0000, 0x5d28ea0, 0x0, 0x0, 0xc4229a0030, 0xc42048e4a0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4241cc940, 0xc42241d440, 0xc42241d3d0, 0x6a4c120, 0xc4229a0000, 0x735a000, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4241cc940, 0xc42241d440, 0xc42241d3d0, 0x6a4c120, 0xc4229a0000, 0x0, 0xc42048e4a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc42241d648, 0x6a551e0, 0xc4241cc940, 0xc42241d440, 0xc42241d3d0, 0xebce21f77a15a783, 0xc4235f9590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc42241d000, 0x6a551e0, 0xc4241cc940, 0xc42241d3d0, 0x1, 0x73e91b0, 0xc421b32cf0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc42241d000, 0x6a551e0, 0xc4241cc940, 0x6a568e0, 0xc4226bb4a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc42241d000, 0x6a551e0, 0xc4241cc940, 0x6a568e0, 0xc4226bb4a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc42241d000, 0x6a551e0, 0xc4241cc940, 0x6a568e0, 0xc4226bb4a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc42241d000, 0x6a551e0, 0xc423229300, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc423229300, 0x0, 0x0, 0xc421691022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423200670, 0xc4204a4700, 0x6a551e0, 0xc423229300, 0xc422fc8c40, 0x5400, 0x15000, 0xc420724a90, 0xc422f7aa10, 0xc423200660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1991 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc4213a5b90, 0xc4202d8b90, 0x7, 0xc423abcf50, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc4213a5b90, 0x7, 0xc423abcf50, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc4213a5b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e64a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc4213a5b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e64a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc4213a5b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e64a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc4213a5b90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e64900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42293bc00, 0x6a552a0, 0xc4213a5a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e64900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc42456e8c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e64900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4218f1ba0, 0x6a551e0, 0xc42456e8c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc4218f1ba0, 0x6a551e0, 0xc42456e8c0, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc4218f1ba0, 0x6a551e0, 0xc42456e8c0, 0xc4218f1ba0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc4218f1ba0, 0x6a551e0, 0xc42456e8c0, 0xc424615bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc42307d000, 0x6a551e0, 0xc42456e8c0, 0x6a568e0, 0xc420c60480, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc42307d000, 0x6a551e0, 0xc42456e8c0, 0x6a568e0, 0xc420c60480, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc42307d000, 0x6a551e0, 0xc4232283c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc4232283c0, 0x0, 0x0, 0xc42252100d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422f41e50, 0xc4204a4700, 0x6a551e0, 0xc4232283c0, 0xc422fd2700, 0x5400, 0x15000, 0xc420724a90, 0xc422bdf1f0, 0xc422f41e40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115229 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423b4dcc0, 0x45563f3, 0xc4221871e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423b4dcc0, 0x5d2c558, 0xc422187420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc423d64870, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc423d64870, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b30438, 0x5b6ec40, 0xc422324060, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424105f80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b30438, 0x5b6ec40, 0xc422324060, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424105f80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b30438, 0x5b6ec40, 0xc422324060, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424105f80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b30438, 0x5b6ec40, 0xc422324060, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f805e0, 0x6a551e0, 0xc424105f80, 0xc423b30438, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7e4894a9b) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420343a40, 0xc423b30420, 0x6a551e0, 0xc424105f80, 0xc400000008, 0xc424918900, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420343a40, 0x6a551e0, 0xc424105f80, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420343a40, 0x6a551e0, 0xc424105f80, 0xc421b53ef0, 0xc420b3d860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 2034 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc423406720, 0xc4202d8b90, 0x7, 0xc42432fd10, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc423406720, 0x7, 0xc42432fd10, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc423406720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc423406720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc423406720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc423406720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e06e00, 0x6a552a0, 0xc4234066c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc424a16280, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420bdbc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422629520, 0x6a551e0, 0xc424a16280, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc424a16280, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc424a16280, 0xc424cff878, 0xc425242300, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422629520, 0x6a551e0, 0xc424a16280, 0xc425242300, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc42241c3d0, 0x6a551e0, 0xc424a16280, 0xc424b93e20, 0x13, 0x20, 0xcbf, 0x6a651a0, 0xc42102e300, 0xc4217d46a0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4237ff170, 0x6a551e0, 0xc424a16280, 0xc42241c440, 0xc42241c3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4237ff170, 0xc422eea1a8, 0xc424d00b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42378c040, 0x5c46757, 0xc, 0x6a4c120, 0xc4237ff170) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42378c040, 0x6a4c120, 0xc4237ff170) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc424a16280, 0x6a4c120, 0xc4237ff170, 0x5d28ea0, 0x0, 0x0, 0xc4237ff1d0, 0xc420358cc0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc424a16280, 0xc42241c440, 0xc42241c3d0, 0x6a4c120, 0xc4237ff170, 0x735a000, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc424a16280, 0xc42241c440, 0xc42241c3d0, 0x6a4c120, 0xc4237ff170, 0x0, 0xc420358cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc42241c648, 0x6a551e0, 0xc424a16280, 0xc42241c440, 0xc42241c3d0, 0x8c189646f815f4b5, 0xc424d01590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc42241c000, 0x6a551e0, 0xc424a16280, 0xc42241c3d0, 0x1, 0x73e91b0, 0xc4213a2480, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc42241c000, 0x6a551e0, 0xc424a16280, 0x6a568e0, 0xc424201260, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc42241c000, 0x6a551e0, 0xc424a16280, 0x6a568e0, 0xc424201260, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc42241c000, 0x6a551e0, 0xc424a16280, 0x6a568e0, 0xc424201260, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc42241c000, 0x6a551e0, 0xc4232290c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc4232290c0, 0x0, 0x0, 0xc421ed6035, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4232005f0, 0xc4204a4700, 0x6a551e0, 0xc4232290c0, 0xc422fc8a80, 0x5400, 0x15000, 0xc420724a90, 0xc422f7a9d0, 0xc4232005e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 1999 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).transportMonitor(0xc422abaa00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:1234 +0x25f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect.func1(0xc422abaa00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:837 +0x1ce created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*addrConn).connect /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/clientconn.go:828 +0xeb goroutine 2051 [IO wait]: internal/poll.runtime_pollWait(0x938e958, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4231b0098, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4231b0098, 0xc422055000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4231b0080, 0xc422055000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4231b0080, 0xc422055000, 0x400, 0x400, 0x2e, 0x8, 0x0) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4205b04f0, 0xc422055000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d crypto/tls.(*block).readFromUntil(0xc4226e5bc0, 0x77aa1a0, 0xc4205b04f0, 0x5, 0xc4205b04f0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f3c700, 0x5d2d117, 0xc421f3c820, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f3c700, 0xc422512000, 0x8000, 0x8000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 bufio.(*Reader).Read(0xc4228f2300, 0xc4226fce38, 0x9, 0x9, 0x9, 0x0, 0x0) /Users/andrei/work/src/go/src/bufio/bufio.go:213 +0x30b io.ReadAtLeast(0x6a34360, 0xc4228f2300, 0xc4226fce38, 0x9, 0x9, 0x9, 0xc422b90de0, 0x400436e, 0xc422b90e8f) /Users/andrei/work/src/go/src/io/io.go:309 +0x86 io.ReadFull(0x6a34360, 0xc4228f2300, 0xc4226fce38, 0x9, 0x9, 0x44f790b, 0xc422745e2c, 0xc422bde860) /Users/andrei/work/src/go/src/io/io.go:327 +0x58 github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.readFrameHeader(0xc4226fce38, 0x9, 0x9, 0x6a34360, 0xc4228f2300, 0x0, 0x0, 0xc422745e20, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:237 +0x7b github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2.(*Framer).ReadFrame(0xc4226fce00, 0xc422745e20, 0xc422745e20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/golang.org/x/net/http2/frame.go:492 +0xa4 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).reader(0xc422eeac00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1173 +0xe1 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:273 +0xc73 goroutine 113694 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42370e240, 0xc4206a4230, 0x7, 0xc4232b1f90, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42370e240, 0x7, 0xc4232b1f90, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42370e240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42370e240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42370e240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42370e240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421a816c0, 0x6a552a0, 0xc42370e1e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42318bd00, 0xc420d5ae00, 0x0, 0xc42318be38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc420d5ae00, 0x1, 0xc42370e180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc422f7de60, 0xc420538600, 0x5, 0x5, 0x15179d56fb9230d0, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4237ff140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 2019 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc4233d8ba0, 0xc4202d8b90, 0x7, 0xc420ae34f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc4233d8ba0, 0x7, 0xc420ae34f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc4233d8ba0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc4233d8ba0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc4233d8ba0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc4233d8ba0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc424838a80, 0x6a552a0, 0xc4233d8b10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc423aa0580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4230da680, 0x6a551e0, 0xc423aa0580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc423aa0580, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc423aa0580, 0xc424a55878, 0xc4236dea00, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc4230da680, 0x6a551e0, 0xc423aa0580, 0xc4236dea00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4231f13d0, 0x6a551e0, 0xc423aa0580, 0xc422682200, 0x13, 0x20, 0xcbe, 0x6a651a0, 0xc421391b00, 0xc424740360, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4236843c0, 0x6a551e0, 0xc423aa0580, 0xc4231f1440, 0xc4231f13d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4236843c0, 0xc422f16c28, 0xc424a56b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc424038bc0, 0x5c46757, 0xc, 0x6a4c120, 0xc4236843c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc424038bc0, 0x6a4c120, 0xc4236843c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc423aa0580, 0x6a4c120, 0xc4236843c0, 0x5d28ea0, 0x0, 0x0, 0xc4236843f0, 0xc42230b3a0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc423aa0580, 0xc4231f1440, 0xc4231f13d0, 0x6a4c120, 0xc4236843c0, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc423aa0580, 0xc4231f1440, 0xc4231f13d0, 0x6a4c120, 0xc4236843c0, 0x0, 0xc42230b3a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc4231f1648, 0x6a551e0, 0xc423aa0580, 0xc4231f1440, 0xc4231f13d0, 0xf377ff892bd87332, 0xc424a57590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc4231f1000, 0x6a551e0, 0xc423aa0580, 0xc4231f13d0, 0x1, 0x73e91b0, 0xc421b0aab0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc4231f1000, 0x6a551e0, 0xc423aa0580, 0x6a568e0, 0xc42252cc00, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc4231f1000, 0x6a551e0, 0xc423aa0580, 0x6a568e0, 0xc42252cc00, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4231f1000, 0x6a551e0, 0xc423aa0580, 0x6a568e0, 0xc42252cc00, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4231f1000, 0x6a551e0, 0xc42302a980, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc42302a980, 0x0, 0x0, 0xc420cfb037, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423006470, 0xc4204a4700, 0x6a551e0, 0xc42302a980, 0xc4224d4700, 0x5400, 0x15000, 0xc420724a90, 0xc422bde4b0, 0xc423006460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 110173 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc424550150, 0xc4206a4230, 0x7, 0xc421f857c0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc424550150, 0x7, 0xc421f857c0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc424550150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc424550150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc424550150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc424550150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42420ac40, 0x6a552a0, 0xc424550120, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42521ef78, 0xc424582000, 0x4, 0xc4222d8040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424582000, 0x5, 0xc4222d8000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420308a00, 0x3c, 0x3c, 0x15179d56b28621f8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420308a00, 0x3c, 0x3c, 0x15179d56b28621f8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423f614d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 4884 [IO wait]: internal/poll.runtime_pollWait(0x77252c0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc423640098, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc423640098, 0xc423145000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc423640080, 0xc423145000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc423640080, 0xc423145000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc421ac92c8, 0xc423145000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc423d48000, 0xc423145000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc42375fcb0, 0x73ccbf8, 0xc423d48000, 0x5, 0xc423d48000, 0xa0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42271ee00, 0x5d2d117, 0xc42271ef20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42271ee00, 0xc422e92000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4226bc500, 0xc422e92000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4236451e0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4236451e0, 0xbe9de8a305ccc978, 0x7a62660e0, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc423645310, 0x6a46020, 0xc4236451e0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc423645180, 0x6a551e0, 0xc423908c80, 0xc4210634f0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc421b67dc0, 0x6a611e0, 0xc42271ee00, 0x0, 0x0, 0xc423735037, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc421b67dc0, 0x6a611e0, 0xc42271ee00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc42375fc80, 0x6a61a80, 0xc423d48000, 0xc422725e20, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc423d48000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc423d48000, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 112494 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423f5cf30, 0xc4206a4230, 0x7, 0xc420bd3950, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423f5cf30, 0x7, 0xc420bd3950, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423f5cf30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423f5cf30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423f5cf30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423f5cf30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c3b880, 0x6a552a0, 0xc423f5cf00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4228caf78, 0xc424a72300, 0x2, 0xc4239108e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424a72300, 0x3, 0xc4239108c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42447c000, 0x44, 0x44, 0x15179d56de644020, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42447c000, 0x44, 0x44, 0x15179d56de644020, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42086f800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 21300 [IO wait]: internal/poll.runtime_pollWait(0x73eb168, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc42414d418, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc42414d418, 0xc42239f000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc42414d400, 0xc42239f000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc42414d400, 0xc42239f000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc423f80f90, 0xc42239f000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f51680, 0xc42239f000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc42275ab10, 0x73ccbf8, 0xc421f51680, 0x5, 0xc421f51680, 0xbe) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42204dc00, 0x5d2d117, 0xc42204dd20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42204dc00, 0xc421b3d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4227bc0c0, 0xc421b3d000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc423635560) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc423635560, 0xbe9de8a321b512f8, 0x7c20e748e, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc423635690, 0x6a46020, 0xc423635560, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc423635500, 0x6a551e0, 0xc420f28c40, 0xc4218a39e0, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc421278800, 0x6a611e0, 0xc42204dc00, 0x0, 0x0, 0xc42389300d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc421278800, 0x6a611e0, 0xc42204dc00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4228baa50, 0x6a61a80, 0xc421f51680, 0xc4219723f0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f51680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc421f51680, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 2024 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc422b20420, 0xc4202d8b90, 0x7, 0xc4239c3680, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc422b20420, 0x7, 0xc4239c3680, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc422b20420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6a500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc422b20420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6a500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc422b20420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6a500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc422b20420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6a400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423d31c00, 0x6a552a0, 0xc422b203c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6a400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc424e874c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420b6a400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4218f01a0, 0x6a551e0, 0xc424e874c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc4218f01a0, 0x6a551e0, 0xc424e874c0, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc4218f01a0, 0x6a551e0, 0xc424e874c0, 0xc4218f01a0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc4218f01a0, 0x6a551e0, 0xc424e874c0, 0xc4239abbb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc4229ea000, 0x6a551e0, 0xc424e874c0, 0x6a568e0, 0xc421f37920, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4229ea000, 0x6a551e0, 0xc424e874c0, 0x6a568e0, 0xc421f37920, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4229ea000, 0x6a551e0, 0xc42302b440, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc42302b440, 0x0, 0x0, 0xc422525022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423006720, 0xc4204a4700, 0x6a551e0, 0xc42302b440, 0xc4224d5c00, 0x5400, 0x15000, 0xc420724a90, 0xc422bde680, 0xc423006700) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115253 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42359d470, 0xc421080cb0, 0xc42395c3c0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359d470, 0x15179d56f8bce8f0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359d470, 0x15179d56f8bce8f0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42359d470, 0x15179d56f8bce8f0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42359d440, 0x15179d56f8bce8f0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42359d290, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42359d290, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42359d260, 0x5c4d11c, 0x10, 0xc423a772c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42359d260, 0xc421080c40, 0xc42359d260, 0xc4230e1568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42359d260, 0xc421080c40, 0xc420b53500, 0xc421ac83a8, 0xc42456fe00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420fb1b00, 0x5c13340, 0xc421080c40, 0x5c13340, 0xc421080c40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420fb1b00, 0x5c13340, 0xc421080c40, 0xc422d4ce20, 0xc422d4ce40, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420fb1b00, 0x5c13340, 0xc421080c40, 0xc422d4ce20, 0xc422d4ce40, 0x6, 0xc4230e1968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420fb1b00, 0xc421080bd0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422b8bc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423875040, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423875040, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423875040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114260 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422027bf0, 0xc4206a4230, 0x7, 0xc4224f0230, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422027bf0, 0x7, 0xc4224f0230, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422027bf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422027bf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422027bf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422027bf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4224d5180, 0x6a552a0, 0xc422027bc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42447af78, 0xc423ea6e00, 0x5, 0xc4222d9050) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423ea6e00, 0x5, 0xc4222d9000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42195c000, 0x44, 0x44, 0x15179d570e5d0ed8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42195c000, 0x44, 0x44, 0x15179d570e5d0ed8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420f65bf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114924 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422fac8c0, 0x45563f3, 0xc4248491e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422fac8c0, 0x5d2c558, 0xc424849420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42377b950, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42377b950, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4211787a8, 0x5b6ec40, 0xc4224a4ae0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4238bfb80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4211787a8, 0x5b6ec40, 0xc4224a4ae0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc4238bfb80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4211787a8, 0x5b6ec40, 0xc4224a4ae0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4238bfb80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4211787a8, 0x5b6ec40, 0xc4224a4ae0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac86c0, 0x6a551e0, 0xc4238bfb80, 0xc4211787a8, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x9364f88) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423d9c2a0, 0xc421178790, 0x6a551e0, 0xc4238bfb80, 0xc400000008, 0xc42403bd00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423d9c2a0, 0x6a551e0, 0xc4238bfb80, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423d9c2a0, 0x6a551e0, 0xc4238bfb80, 0xc420b99080, 0xc4224a4a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 2052 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.loopyWriter(0x6a551e0, 0xc42302a5c0, 0xc4226bee70, 0xc42212efb8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:727 +0x3c6 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client.func3(0xc422eeac00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:305 +0x5e created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:304 +0xdc4 goroutine 2053 [select]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*http2Client).keepalive(0xc422eeac00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:1350 +0x305 created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.newHTTP2Client /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/http2_client.go:309 +0xe44 goroutine 114570 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4211e1bc0, 0x15179d56fad1e640, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e9d000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4211e1bc0, 0x15179d56fad1e640, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e9d000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4211e1bc0, 0x15179d56fad1e640, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e9d000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4211e1b90, 0x15179d56fad1e640, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e9d000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4211e1b30, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e9d000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4211e1b30, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4211e1b30, 0x5c4d11c, 0x10, 0xc42191d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4211e1b30, 0xc422f13cf8, 0xc4211e1b30, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4211e1b30, 0xc422f13cf8, 0x1f, 0xc420bcb630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421246690, 0xc422f13ce0, 0x6a551e0, 0xc422a5bb00, 0xc400000008, 0xc42403a240, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421246690, 0x6a551e0, 0xc422a5bb00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421246690, 0x6a551e0, 0xc422a5bb00, 0xc422f13a20, 0xc423409320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114404 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4224e0d20, 0xc4206a4230, 0x7, 0xc42382c320, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4224e0d20, 0x7, 0xc42382c320, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4224e0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4224e0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4224e0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4224e0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4237cfdc0, 0x6a552a0, 0xc4224e0cf0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424e32f78, 0xc424f6f500, 0x2, 0xc42141ce60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424f6f500, 0x3, 0xc42141ce40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42195c000, 0x46, 0x46, 0x15179d5710b2ce48, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42195c000, 0x46, 0x46, 0x15179d5710b2ce48, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42275a1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115183 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc422427830, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422427830, 0x15179d57215a4f50, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422427830, 0x15179d57215a4f50, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422427830, 0x15179d57215a4f50, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422427800, 0x15179d57215a4f50, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224277a0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224277a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224277a0, 0x5c4d11c, 0x10, 0xc42401b9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224277a0, 0xc422c75988, 0xc4224277a0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224277a0, 0xc422c75988, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4210c7340, 0xc422c75970, 0x6a551e0, 0xc424104100, 0xc400000008, 0xc422e98600, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4210c7340, 0x6a551e0, 0xc424104100, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4210c7340, 0x6a551e0, 0xc424104100, 0xc422c758c0, 0xc421a1dce0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 2202 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc42377a930, 0xc420518e60, 0x7, 0xc420e2b720, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc42377a930, 0x7, 0xc420e2b720, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc42377a930, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422411d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc42377a930, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422411d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc42377a930, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422411d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc42377a930, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4208a7300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c3ba40, 0x6a552a0, 0xc4239190e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4208a7300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc4238be4c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4208a7300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423614ea0, 0x6a551e0, 0xc4238be4c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc423614ea0, 0x6a551e0, 0xc4238be4c0, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc423614ea0, 0x6a551e0, 0xc4238be4c0, 0xc423614ea0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc423614ea0, 0x6a551e0, 0xc4238be4c0, 0xc42383bbb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc421249000, 0x6a551e0, 0xc4238be4c0, 0x6a568e0, 0xc4238015c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421249000, 0x6a551e0, 0xc4238be4c0, 0x6a568e0, 0xc4238015c0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421249000, 0x6a551e0, 0xc42101d080, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc42101d080, 0x0, 0x0, 0xc4226cc037, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc42034fa20, 0xc42100a000, 0x6a551e0, 0xc42101d080, 0xc420cd16c0, 0x5400, 0x15000, 0xc42136a190, 0xc421b3f740, 0xc42034fa00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114001 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc422fb5ef0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422fb5ef0, 0x15179d5705edc580, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422fb5ef0, 0x15179d5705edc580, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422fb5ef0, 0x15179d5705edc580, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422fb5ec0, 0x15179d5705edc580, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422fb5e60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422fb5e60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422fb5e60, 0x5c4d11c, 0x10, 0xc42314d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422fb5e60, 0xc42238f6c8, 0xc422fb5e60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422fb5e60, 0xc42238f6c8, 0x1f, 0xc42002f630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423a96230, 0xc42238f6b0, 0x6a551e0, 0xc424909e40, 0xc400000008, 0xc424265440, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423a96230, 0x6a551e0, 0xc424909e40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423a96230, 0x6a551e0, 0xc424909e40, 0xc42238f3f0, 0xc4226bb800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113359 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420732ab0, 0xc4206a4230, 0x7, 0xc42484e5a0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420732ab0, 0x7, 0xc42484e5a0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420732ab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420732ab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420732ab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420732ab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4245af180, 0x6a552a0, 0xc420732a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421fbcf78, 0xc424583500, 0x0, 0xc42164fc00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424583500, 0x1, 0xc42164fc00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4218c3b00, 0x30, 0x30, 0x15179d56f6ef87a8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4218c3b00, 0x30, 0x30, 0x15179d56f6ef87a8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4207329c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114982 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc420bd5140, 0xc4202d8b90, 0x7, 0xc421989180, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc420bd5140, 0x7, 0xc421989180, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc420bd5140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc420bd5140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc420bd5140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc420bd5140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc42117d880, 0x6a552a0, 0xc420f65cb0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc42117d880, 0x6a552a0, 0xc420f65cb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc420f65500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc420f65500, 0xc421009b30, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc421621620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 2086 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4228bac90, 0xc4206a4230, 0x7, 0xc423c33c20, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4228bac90, 0x7, 0xc423c33c20, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4228bac90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4228bac90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4228bac90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4228bac90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4247c41c0, 0x6a552a0, 0xc4228bac00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4238bf640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422040680, 0x6a551e0, 0xc4238bf640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4238bf640, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4238bf640, 0xc423ceb878, 0xc423b87500, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422040680, 0x6a551e0, 0xc4238bf640, 0xc423b87500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4210c13d0, 0x6a551e0, 0xc4238bf640, 0xc42250f5e0, 0x13, 0x20, 0xcc7, 0x6a651a0, 0xc422718000, 0xc4246028b0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc420267ef0, 0x6a551e0, 0xc4238bf640, 0xc4210c1440, 0xc4210c13d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc420267ef0, 0xc422eeb6a8, 0xc423cecb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc422cee180, 0x5c46757, 0xc, 0x6a4c120, 0xc420267ef0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc422cee180, 0x6a4c120, 0xc420267ef0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4238bf640, 0x6a4c120, 0xc420267ef0, 0x5d28ea0, 0x0, 0x0, 0xc420267f20, 0xc420fceec0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4238bf640, 0xc4210c1440, 0xc4210c13d0, 0x6a4c120, 0xc420267ef0, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4238bf640, 0xc4210c1440, 0xc4210c13d0, 0x6a4c120, 0xc420267ef0, 0x0, 0xc420fceec0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc4210c1648, 0x6a551e0, 0xc4238bf640, 0xc4210c1440, 0xc4210c13d0, 0xced8126297f6c2b0, 0xc423ced590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc4210c1000, 0x6a551e0, 0xc4238bf640, 0xc4210c13d0, 0x1, 0x73e91b0, 0xc4239359e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc4210c1000, 0x6a551e0, 0xc4238bf640, 0x6a568e0, 0xc4224a41e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc4210c1000, 0x6a551e0, 0xc4238bf640, 0x6a568e0, 0xc4224a41e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4210c1000, 0x6a551e0, 0xc4238bf640, 0x6a568e0, 0xc4224a41e0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4210c1000, 0x6a551e0, 0xc421154900, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc421154900, 0x0, 0x0, 0xc42248a037, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc420394760, 0xc42100a460, 0x6a551e0, 0xc421154900, 0xc420cd01c0, 0x5400, 0x15000, 0xc4206d5810, 0xc422c1cd80, 0xc420394740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114342 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbe6f0, 0x15179d56f5fb2140, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e65b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbe6f0, 0x15179d56f5fb2140, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e65b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423fbe6f0, 0x15179d56f5fb2140, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e65b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423fbe6c0, 0x15179d56f5fb2140, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e65b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423fbe630, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420e65b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423fbe630, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423fbe600, 0x5c4d11c, 0x10, 0xc422b4d2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423fbe600, 0xc422666230, 0xc423fbe600, 0xc423cc3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423fbe600, 0xc422666230, 0xc420b53500, 0xc423f80a68, 0xc422495040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423fbe0f0, 0x5c13340, 0xc422666230, 0x5c13340, 0xc422666230, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423fbe0f0, 0x5c13340, 0xc422666230, 0xc422d4d4c0, 0xc422d4d4e0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423fbe0f0, 0x5c13340, 0xc422666230, 0xc422d4d4c0, 0xc422d4d4e0, 0x6, 0xc423cc3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423fbe0f0, 0xc4226661c0, 0xc420eaab00, 0x0, 0x0, 0x3f, 0xc422b8dc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422c38b40, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422c38b40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422c38b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 108571 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423e77cb0, 0xc4206a4230, 0x7, 0xc421641b30, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423e77cb0, 0x7, 0xc421641b30, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423e77cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423e77cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423e77cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423e77cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e6a1c0, 0x6a552a0, 0xc423e77c80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4224b4f78, 0xc421e2f500, 0x6, 0xc422d9e9e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421e2f500, 0x5, 0xc422d9e980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc425188000, 0x43, 0x43, 0x15179d568fea8378, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc425188000, 0x43, 0x43, 0x15179d568fea8378, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423e77a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114878 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4219f7350, 0xc4206a4230, 0x7, 0xc4218dd4f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4219f7350, 0x7, 0xc4218dd4f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4219f7350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4219f7350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4219f7350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4219f7350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c268c0, 0x6a552a0, 0xc4219f7320, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4224b5d00, 0xc4236af500, 0x0, 0xc4224b5e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4236af500, 0x1, 0xc4219f7260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4226f97a0, 0xc420c86000, 0x5, 0x5, 0x15179d571c38c1c8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42375fe30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114935 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc42387bb00, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387bb00, 0x15179d571acd6e10, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387bb00, 0x15179d571acd6e10, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42387bb00, 0x15179d571acd6e10, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42387baa0, 0x15179d571acd6e10, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42387ba40, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42387ba40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42387ba40, 0x5c4d11c, 0x10, 0xc4249ad9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42387ba40, 0xc4237dadd8, 0xc42387ba40, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42387ba40, 0xc4237dadd8, 0x1f, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420b61ce0, 0xc4237dadc0, 0x6a551e0, 0xc424a16a40, 0xc400000008, 0xc421450c40, 0x299a73a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420b61ce0, 0x6a551e0, 0xc424a16a40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420b61ce0, 0x6a551e0, 0xc424a16a40, 0xc4237dad10, 0xc4200cdc80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 2121 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4211e0f90, 0xc4206a4230, 0x7, 0xc4224f06e0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4211e0f90, 0x7, 0xc4224f06e0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4211e0f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a0500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4211e0f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a0500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4211e0f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a0500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4211e0f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a0400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42293b500, 0x6a552a0, 0xc4211e0e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a0400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4241cc5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a0400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423709040, 0x6a551e0, 0xc4241cc5c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc423709040, 0x6a551e0, 0xc4241cc5c0, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc423709040, 0x6a551e0, 0xc4241cc5c0, 0xc423709040, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc423709040, 0x6a551e0, 0xc4241cc5c0, 0xc4239e1bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc421248000, 0x6a551e0, 0xc4241cc5c0, 0x6a568e0, 0xc423a10d20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421248000, 0x6a551e0, 0xc4241cc5c0, 0x6a568e0, 0xc423a10d20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421248000, 0x6a551e0, 0xc42104b580, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc42104b580, 0x0, 0x0, 0xc421e9000d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423201de0, 0xc42100a460, 0x6a551e0, 0xc42104b580, 0xc4232268c0, 0x5400, 0x15000, 0xc4206d5810, 0xc421fd8170, 0xc423201dd0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114422 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4224e0f00, 0xc42299fc70, 0xc4236da120, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0f00, 0x15179d5710ed9528, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0f00, 0x15179d5710ed9528, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224e0f00, 0x15179d5710ed9528, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224e0ea0, 0x15179d5710ed9528, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224e0e40, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224e0e40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224e0e40, 0x5c4d11c, 0x10, 0xc4236f19b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224e0e40, 0xc422c82a68, 0xc4224e0e40, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224e0e40, 0xc422c82a68, 0x1f, 0xc420033630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42299fc00, 0xc422c82a50, 0x6a551e0, 0xc42141cf00, 0xc400000008, 0xc4231b66c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42299fc00, 0x6a551e0, 0xc42141cf00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42299fc00, 0x6a551e0, 0xc42141cf00, 0xc422c829a0, 0xc4236da0c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 2073 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc420c4b860, 0xc420518e60, 0x7, 0xc422286820, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc420c4b860, 0x7, 0xc422286820, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc420c4b860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc420c4b860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc420c4b860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc420c4b860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421acea80, 0x6a552a0, 0xc420c4b5f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc423e24e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422058700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422041860, 0x6a551e0, 0xc423e24e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc422041860, 0x6a551e0, 0xc423e24e40, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc422041860, 0x6a551e0, 0xc423e24e40, 0xc422041860, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc422041860, 0x6a551e0, 0xc423e24e40, 0xc4248b5bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc4213ca000, 0x6a551e0, 0xc423e24e40, 0x6a568e0, 0xc4236a5440, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4213ca000, 0x6a551e0, 0xc423e24e40, 0x6a568e0, 0xc4236a5440, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4213ca000, 0x6a551e0, 0xc42132ad40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc42132ad40, 0x0, 0x0, 0xc42248b037, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423201240, 0xc42100a000, 0x6a551e0, 0xc42132ad40, 0xc422fc9dc0, 0x5400, 0x15000, 0xc42136a190, 0xc422d482b0, 0xc423201230) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 2074 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc423b9b1d0, 0xc420518e60, 0x7, 0xc422be7f40, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc423b9b1d0, 0x7, 0xc422be7f40, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc423b9b1d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423acea00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc423b9b1d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423acea00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc423b9b1d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423acea00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc423b9b1d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4208a6200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc424b9cc40, 0x6a552a0, 0xc423b9b140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4208a6200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc424e87780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4208a6200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4218f0340, 0x6a551e0, 0xc424e87780, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc4218f0340, 0x6a551e0, 0xc424e87780, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc4218f0340, 0x6a551e0, 0xc424e87780, 0xc4218f0340, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc4218f0340, 0x6a551e0, 0xc424e87780, 0xc422a27bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc4210c0000, 0x6a551e0, 0xc424e87780, 0x6a568e0, 0xc421f37ce0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4210c0000, 0x6a551e0, 0xc424e87780, 0x6a568e0, 0xc421f37ce0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4210c0000, 0x6a551e0, 0xc42132b180, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc42132b180, 0x0, 0x0, 0xc421c3300d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4232012f0, 0xc42100a000, 0x6a551e0, 0xc42132b180, 0xc423226000, 0x5400, 0x15000, 0xc42136a190, 0xc422d48350, 0xc4232012e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 2075 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc420f649f0, 0xc420518e60, 0x7, 0xc42484e1e0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc420f649f0, 0x7, 0xc42484e1e0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc420f649f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4210fe800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc420f649f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4210fe800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc420f649f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4210fe800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc420f649f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4210fe600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4203d5c00, 0x6a552a0, 0xc420f64930, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4210fe600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc4226c5540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4210fe600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422bf3a00, 0x6a551e0, 0xc4226c5540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4226c5540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4226c5540, 0xc423257878, 0xc423de2300, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422bf3a00, 0x6a551e0, 0xc4226c5540, 0xc423de2300, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc42204a3d0, 0x6a551e0, 0xc4226c5540, 0xc4224914c0, 0x13, 0x20, 0xcbc, 0x6a651a0, 0xc423945800, 0xc42437f820, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc421f525d0, 0x6a551e0, 0xc4226c5540, 0xc42204a440, 0xc42204a3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc421f525d0, 0xc421625828, 0xc423258b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc423910000, 0x5c46757, 0xc, 0x6a4c120, 0xc421f525d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc423910000, 0x6a4c120, 0xc421f525d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4226c5540, 0x6a4c120, 0xc421f525d0, 0x5d28ea0, 0x0, 0x0, 0xc421f52600, 0xc421c2c760, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4226c5540, 0xc42204a440, 0xc42204a3d0, 0x6a4c120, 0xc421f525d0, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4226c5540, 0xc42204a440, 0xc42204a3d0, 0x6a4c120, 0xc421f525d0, 0x0, 0xc421c2c760) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc42204a648, 0x6a551e0, 0xc4226c5540, 0xc42204a440, 0xc42204a3d0, 0x1ba67057083f6ad2, 0xc423259590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc42204a000, 0x6a551e0, 0xc4226c5540, 0xc42204a3d0, 0x1, 0x73e91b0, 0xc422d68120, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc42204a000, 0x6a551e0, 0xc4226c5540, 0x6a568e0, 0xc422227ec0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc42204a000, 0x6a551e0, 0xc4226c5540, 0x6a568e0, 0xc422227ec0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc42204a000, 0x6a551e0, 0xc4226c5540, 0x6a568e0, 0xc422227ec0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc42204a000, 0x6a551e0, 0xc42132b480, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc42132b480, 0x0, 0x0, 0xc422c3100d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423201390, 0xc42100a000, 0x6a551e0, 0xc42132b480, 0xc4232261c0, 0x5400, 0x15000, 0xc42136a190, 0xc422d483e0, 0xc423201380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 2076 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc4224f3350, 0xc420518e60, 0x7, 0xc4243e18b0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc4224f3350, 0x7, 0xc4243e18b0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc4224f3350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420f72600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc4224f3350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420f72600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc4224f3350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420f72600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc4224f3350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ee48c0, 0x6a552a0, 0xc4224f32c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc424dce980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4212a7f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422b12b60, 0x6a551e0, 0xc424dce980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc422b12b60, 0x6a551e0, 0xc424dce980, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc422b12b60, 0x6a551e0, 0xc424dce980, 0xc422b12b60, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc422b12b60, 0x6a551e0, 0xc424dce980, 0xc42525fbb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc4213cb000, 0x6a551e0, 0xc424dce980, 0x6a568e0, 0xc422980060, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4213cb000, 0x6a551e0, 0xc424dce980, 0x6a568e0, 0xc422980060, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4213cb000, 0x6a551e0, 0xc42132b840, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc42132b840, 0x0, 0x0, 0xc4224c6022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423201450, 0xc42100a000, 0x6a551e0, 0xc42132b840, 0xc423226380, 0x5400, 0x15000, 0xc42136a190, 0xc422d484b0, 0xc423201440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 108270 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42387b1a0, 0xc4206a4230, 0x7, 0xc4212ba050, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42387b1a0, 0x7, 0xc4212ba050, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42387b1a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42387b1a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42387b1a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42387b1a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc424b9cfc0, 0x6a552a0, 0xc42387b170, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423232f78, 0xc424f77c00, 0x9, 0xc4208a7790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424f77c00, 0x9, 0xc4208a7700) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x3d, 0x3d, 0x15179d568b6b48a0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x3d, 0x3d, 0x15179d568b6b48a0, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42387b0e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 7292 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423292de0, 0xc4206a4230, 0x7, 0xc4210fcff0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423292de0, 0x7, 0xc4210fcff0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423292de0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4200afe00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423292de0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4200afe00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423292de0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4200afe00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423292de0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4200afd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c3a540, 0x6a552a0, 0xc423292d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4200afd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc424a17680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc4200afd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422b13040, 0x6a551e0, 0xc424a17680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc422b13040, 0x6a551e0, 0xc424a17680, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc422b13040, 0x6a551e0, 0xc424a17680, 0xc422b13040, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc422b13040, 0x6a551e0, 0xc424a17680, 0xc4234c5bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc4229eb000, 0x6a551e0, 0xc424a17680, 0x6a568e0, 0xc42245ad20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4229eb000, 0x6a551e0, 0xc424a17680, 0x6a568e0, 0xc42245ad20, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4229eb000, 0x6a551e0, 0xc4238a9d00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc4238a9d00, 0x0, 0x0, 0xc421eea00d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc423006110, 0xc42100a460, 0x6a551e0, 0xc4238a9d00, 0xc423eca700, 0x5400, 0x15000, 0xc4206d5810, 0xc422bebf10, 0xc423006100) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114713 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124ebd0, 0x15179d57170c04f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124ebd0, 0x15179d57170c04f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42124ebd0, 0x15179d57170c04f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224f5800, 0x15179d57170c04f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224f5590, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224f5590, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224f5590, 0x5c4d11c, 0x10, 0xc423a019b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224f5590, 0xc423873828, 0xc4224f5590, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224f5590, 0xc423873828, 0x1f, 0xc42072de30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4225eb730, 0xc423873810, 0x6a551e0, 0xc42403bac0, 0xc400000008, 0xc4232a4340, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4225eb730, 0x6a551e0, 0xc42403bac0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4225eb730, 0x6a551e0, 0xc42403bac0, 0xc423873760, 0xc423cd1b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114683 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423753cc0, 0x45563f3, 0xc4216e31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423753cc0, 0x5d2c558, 0xc4216e3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4234067b0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4234067b0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4232704e8, 0x5b6ec40, 0xc421e97e00, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc422281b00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4232704e8, 0x5b6ec40, 0xc421e97e00, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc422281b00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4232704e8, 0x5b6ec40, 0xc421e97e00, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc422281b00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4232704e8, 0x5b6ec40, 0xc421e97e00, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80f20, 0x6a551e0, 0xc422281b00, 0xc4232704e8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x800a6d430) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420b60460, 0xc4232704d0, 0x6a551e0, 0xc422281b00, 0xc400000008, 0xc4224958c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420b60460, 0x6a551e0, 0xc422281b00, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420b60460, 0x6a551e0, 0xc422281b00, 0xc4223b9ef0, 0xc421e97c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114331 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42124cf60, 0xc4206a4230, 0x7, 0xc42350ad70, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42124cf60, 0x7, 0xc42350ad70, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42124cf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420720000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42124cf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420720000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42124cf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420720000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42124cf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420f2de00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc423e2d6c0, 0x6a552a0, 0xc421116840, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc423e2d6c0, 0x6a552a0, 0xc421116840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc420fafc80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420fafc80, 0xc422768780, 0x29, 0x6a652e0, 0xc420b88320, 0xc4203c7380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 115035 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc421117680, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421117680, 0x15179d571d6877c8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421117680, 0x15179d571d6877c8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421117680, 0x15179d571d6877c8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4218bd470, 0x15179d571d6877c8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4218bd3e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4218bd3e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4218bd350, 0x5c4d11c, 0x10, 0xc422ee32c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4218bd350, 0xc4202dbc70, 0xc4218bd350, 0xc423cc3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4218bd350, 0xc4202dbc70, 0xc420b53500, 0xc423f808a8, 0xc423e25b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4218bd230, 0x5c13340, 0xc4202dbc70, 0x5c13340, 0xc4202dbc70, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4218bd230, 0x5c13340, 0xc4202dbc70, 0xc424bd62a0, 0xc424bd62c0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4218bd230, 0x5c13340, 0xc4202dbc70, 0xc424bd62a0, 0xc424bd62c0, 0x6, 0xc423cc3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4218bd230, 0xc4202dbc00, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc421925c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422a7b900, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422a7b900, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422a7b900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114450 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421685d40, 0xc4206a4230, 0x7, 0xc420e80320, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421685d40, 0x7, 0xc420e80320, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421685d40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421685d40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421685d40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421685d40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42209afc0, 0x6a552a0, 0xc421685d10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421ac3d00, 0xc42235b500, 0x0, 0xc421ac3e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42235b500, 0x1, 0xc421685c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421eb9680, 0xc421bc0900, 0x5, 0x5, 0x15179d57115ba108, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420163800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114410 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42275b110, 0x15179d5711b48ac0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42275b110, 0x15179d5711b48ac0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42275b110, 0x15179d5711b48ac0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42275b080, 0x15179d5711b48ac0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42275b020, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42275b020, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42275b020, 0x5c4d11c, 0x10, 0xc423e079b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42275b020, 0xc42371e858, 0xc42275b020, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42275b020, 0xc42371e858, 0x1f, 0xc42072de30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f709a0, 0xc42371e840, 0x6a551e0, 0xc424264880, 0xc400000008, 0xc424918140, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f709a0, 0x6a551e0, 0xc424264880, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f709a0, 0x6a551e0, 0xc424264880, 0xc42371e790, 0xc421a1c900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114635 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42380ac00, 0x15179d56fb4dbd60, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42380ac00, 0x15179d56fb4dbd60, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42380ac00, 0x15179d56fb4dbd60, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42380aba0, 0x15179d56fb4dbd60, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42380aa80, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42380aa80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42380aa50, 0x5c4d11c, 0x10, 0xc423ac52c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42380aa50, 0xc423d9db20, 0xc42380aa50, 0xc424be3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42380aa50, 0xc423d9db20, 0xc420b53500, 0xc4241ca728, 0xc424dced80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc42380a8a0, 0x5c13340, 0xc423d9db20, 0x5c13340, 0xc423d9db20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc42380a8a0, 0x5c13340, 0xc423d9db20, 0xc420358160, 0xc420358180, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc42380a8a0, 0x5c13340, 0xc423d9db20, 0xc420358160, 0xc420358180, 0x6, 0xc424be3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc42380a8a0, 0xc423d9dab0, 0xc420eaab00, 0x0, 0x0, 0xc42193fb00, 0xc420dc59e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4243f63c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4243f63c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4243f63c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114869 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4237b6450, 0xc4206a4230, 0x7, 0xc421f84730, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4237b6450, 0x7, 0xc421f84730, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4237b6450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4237b6450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4237b6450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4237b6450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423072380, 0x6a552a0, 0xc4237b6420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423b39d00, 0xc4221bf100, 0x0, 0xc423b39e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4221bf100, 0x1, 0xc4237b63c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42399dc20, 0xc421bab200, 0x5, 0x5, 0x15179d571a727cd0, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42124f860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 7207 [IO wait]: internal/poll.runtime_pollWait(0x77249c0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4231fb118, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4231fb118, 0xc42348b800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4231fb100, 0xc42348b800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4231fb100, 0xc42348b800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4204f99d0, 0xc42348b800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc420e09a40, 0xc42348b800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc42124e810, 0x73ccbf8, 0xc420e09a40, 0x5, 0xc420e09a40, 0xc6) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422105500, 0x5d2d117, 0xc422105620, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422105500, 0xc4223ca000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421ebfbc0, 0xc4223ca000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc423eca760) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc423eca760, 0xbe9de8a31aa770c8, 0x7bb00dd18, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc423eca890, 0x6a46020, 0xc423eca760, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc423eca700, 0x6a551e0, 0xc4238a9cc0, 0xc4230060e0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420fefe00, 0x6a611e0, 0xc422105500, 0x0, 0x0, 0xc421eea00d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc420fefe00, 0x6a611e0, 0xc422105500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc42124e7e0, 0x6a61a80, 0xc420e09a40, 0xc4203cc550, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc420e09a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc420e09a40, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 114330 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc423d64150, 0xc4239398f0, 0xc42414ed20, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d64150, 0x15179d570f21aec8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d64150, 0x15179d570f21aec8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423d64150, 0x15179d570f21aec8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420fafa70, 0x15179d570f21aec8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420fafa10, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420fafa10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420fafa10, 0x5c4d11c, 0x10, 0xc422b4d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420fafa10, 0xc422a79148, 0xc420fafa10, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420fafa10, 0xc422a79148, 0x1f, 0xc420892630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423939810, 0xc422a79130, 0x6a551e0, 0xc4241d3840, 0xc400000008, 0xc424919d00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423939810, 0x6a551e0, 0xc4241d3840, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423939810, 0x6a551e0, 0xc4241d3840, 0xc422a79080, 0xc42414ec00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114600 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4228d0510, 0xc423552bd0, 0xc4224a52c0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4228d0510, 0x15179d57156ff898, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4228d0510, 0x15179d57156ff898, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4228d0510, 0x15179d57156ff898, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4228d0450, 0x15179d57156ff898, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4228d03f0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4228d03f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4228d03f0, 0x5c4d11c, 0x10, 0xc4234139b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4228d03f0, 0xc420c75358, 0xc4228d03f0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4228d03f0, 0xc420c75358, 0x1f, 0xc420bc8e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423552a80, 0xc420c75340, 0x6a551e0, 0xc424038100, 0xc400000008, 0xc423910a40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423552a80, 0x6a551e0, 0xc424038100, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423552a80, 0x6a551e0, 0xc424038100, 0xc420c75130, 0xc4224a5200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114855 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423c06140, 0x45563f3, 0xc4227871e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423c06140, 0x5d2c558, 0xc422787420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc422b204b0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc422b204b0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4216324e8, 0x5b6ec40, 0xc421bea660, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424104b00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4216324e8, 0x5b6ec40, 0xc421bea660, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc424104b00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4216324e8, 0x5b6ec40, 0xc421bea660, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424104b00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4216324e8, 0x5b6ec40, 0xc421bea660, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca130, 0x6a551e0, 0xc424104b00, 0xc4216324e8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x4015826) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212802a0, 0xc4216324d0, 0x6a551e0, 0xc424104b00, 0xc400000008, 0xc424038a40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212802a0, 0x6a551e0, 0xc424104b00, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212802a0, 0x6a551e0, 0xc424104b00, 0xc4216320b0, 0xc421bea600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114817 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc424597560, 0xc420518e60, 0x7, 0xc4234e71d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc424597560, 0x7, 0xc4234e71d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc424597560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc424597560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc424597560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc424597560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2bb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc42420a380, 0x6a552a0, 0xc42124eff0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc42420a380, 0x6a552a0, 0xc42124eff0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc42124e5d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc42124e5d0, 0xc424209980, 0x29, 0x6a652e0, 0xc420358700, 0xc421228550) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114815 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420f2ef30, 0x15179d5718df3688, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420f2ef30, 0x15179d5718df3688, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420f2ef30, 0x15179d5718df3688, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420f2ef00, 0x15179d5718df3688, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420f2ee70, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420f2ee70, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420f2ee40, 0x5c4d11c, 0x10, 0xc4239bd2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420f2ee40, 0xc423f500e0, 0xc420f2ee40, 0xc4245a5568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420f2ee40, 0xc423f500e0, 0xc420b53500, 0xc423f80528, 0xc424264ac0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420f2ecf0, 0x5c13340, 0xc423f500e0, 0x5c13340, 0xc423f500e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420f2ecf0, 0x5c13340, 0xc423f500e0, 0xc4223d6940, 0xc4223d69c0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420f2ecf0, 0x5c13340, 0xc423f500e0, 0xc4223d6940, 0xc4223d69c0, 0x6, 0xc4245a5968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420f2ecf0, 0xc423f50070, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424922c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42473da40, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42473da40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42473da40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115242 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4233a9040, 0x45563f3, 0xc422c951e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4233a9040, 0x5d2c558, 0xc422c95420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc423009980, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc423009980, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474f4b8, 0x5b6ec40, 0xc422227680, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42335bec0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474f4b8, 0x5b6ec40, 0xc422227680, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc42335bec0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474f4b8, 0x5b6ec40, 0xc422227680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42335bec0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474f4b8, 0x5b6ec40, 0xc422227680, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f8770, 0x6a551e0, 0xc42335bec0, 0xc42474f4b8, 0x0, 0x0, 0x0, 0x1, 0x5a16d20, 0xc4224e6788) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5c310, 0xc42474f4a0, 0x6a551e0, 0xc42335bec0, 0xc400000008, 0xc42335bf80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5c310, 0x6a551e0, 0xc42335bec0, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5c310, 0x6a551e0, 0xc42335bec0, 0xc42474f3f0, 0xc422227560) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115245 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc421a3bc50, 0xc420e5cc40, 0xc4237b04e0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a3bc50, 0x15179d570a1a0240, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a3bc50, 0x15179d570a1a0240, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421a3bc50, 0x15179d570a1a0240, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421a3bbf0, 0x15179d570a1a0240, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421a3bb90, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421a3bb90, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421a3bb90, 0x5c4d11c, 0x10, 0xc424a639b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421a3bb90, 0xc42474f828, 0xc421a3bb90, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421a3bb90, 0xc42474f828, 0x1f, 0xc42002fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5cbd0, 0xc42474f810, 0x6a551e0, 0xc42397c9c0, 0xc400000008, 0xc42397ca80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5cbd0, 0x6a551e0, 0xc42397c9c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5cbd0, 0x6a551e0, 0xc42397c9c0, 0xc42474f760, 0xc4237b01e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114638 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4237feed0, 0x15179d57161e68d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4237feed0, 0x15179d57161e68d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4237feed0, 0x15179d57161e68d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4237fee70, 0x15179d57161e68d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4237fee10, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4237fee10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4237fee10, 0x5c4d11c, 0x10, 0xc4236ad9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4237fee10, 0xc4241c40c8, 0xc4237fee10, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4237fee10, 0xc4241c40c8, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4211f0af0, 0xc4241c40b0, 0x6a551e0, 0xc424dcff80, 0xc400000008, 0xc4249190c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4211f0af0, 0x6a551e0, 0xc424dcff80, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4211f0af0, 0x6a551e0, 0xc424dcff80, 0xc4241c4000, 0xc4211c1440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113731 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42370e3c0, 0x15179d56fbbae2c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42370e3c0, 0x15179d56fbbae2c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42370e3c0, 0x15179d56fbbae2c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42370e390, 0x15179d56fbbae2c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42370e330, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42370e330, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42370e330, 0x5c4d11c, 0x10, 0xc42329f9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42370e330, 0xc422220d28, 0xc42370e330, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42370e330, 0xc422220d28, 0x1f, 0xc420895630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42102c230, 0xc422220d10, 0x6a551e0, 0xc424a17340, 0xc400000008, 0xc421452700, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42102c230, 0x6a551e0, 0xc424a17340, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42102c230, 0x6a551e0, 0xc424a17340, 0xc422220840, 0xc42252c5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114455 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc42348c780, 0x45563f3, 0xc4241491e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc42348c780, 0x5d2c558, 0xc424149420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4204e0900, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4204e0900, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c83f08, 0x5b6ec40, 0xc422308780, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42160f780, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c83f08, 0x5b6ec40, 0xc422308780, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc42160f780, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c83f08, 0x5b6ec40, 0xc422308780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42160f780, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c83f08, 0x5b6ec40, 0xc422308780, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac8398, 0x6a551e0, 0xc42160f780, 0xc422c83f08, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7ae49937f) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422306a80, 0xc422c83ef0, 0x6a551e0, 0xc42160f780, 0xc400000008, 0xc42439c840, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422306a80, 0x6a551e0, 0xc42160f780, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422306a80, 0x6a551e0, 0xc42160f780, 0xc422c83e40, 0xc422308720) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114173 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc42387b650, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387b650, 0x15179d570b621a70, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387b650, 0x15179d570b621a70, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42387b650, 0x15179d570b621a70, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42387b5f0, 0x15179d570b621a70, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42387b590, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42387b590, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42387b590, 0x5c4d11c, 0x10, 0xc4239739b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42387b590, 0xc42296ea68, 0xc42387b590, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42387b590, 0xc42296ea68, 0x1f, 0xc420892630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42454f7a0, 0xc42296ea50, 0x6a551e0, 0xc424a16980, 0xc400000008, 0xc422e315c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42454f7a0, 0x6a551e0, 0xc424a16980, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42454f7a0, 0x6a551e0, 0xc424a16980, 0xc423cd4e70, 0xc4224a4900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114944 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42375f890, 0xc4223cefc0, 0xc42193f200, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42375f890, 0x15179d571b35a5c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42375f890, 0x15179d571b35a5c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42375f890, 0x15179d571b35a5c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4237738f0, 0x15179d571b35a5c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423773890, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423773890, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423773890, 0x5c4d11c, 0x10, 0xc4238299b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423773890, 0xc422220648, 0xc423773890, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423773890, 0xc422220648, 0x1f, 0xc42002fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42454f3b0, 0xc422220630, 0x6a551e0, 0xc424a17cc0, 0xc400000008, 0xc420f29dc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42454f3b0, 0x6a551e0, 0xc424a17cc0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42454f3b0, 0x6a551e0, 0xc424a17cc0, 0xc4222200b0, 0xc4219ce180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114468 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc42259f540, 0x45563f3, 0xc42366d1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc42259f540, 0x5d2c558, 0xc42366d420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc423b4e4b0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc423b4e4b0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421b534b8, 0x5b6ec40, 0xc423d62a80, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42335b600, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421b534b8, 0x5b6ec40, 0xc423d62a80, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc42335b600, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421b534b8, 0x5b6ec40, 0xc423d62a80, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42335b600, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421b534b8, 0x5b6ec40, 0xc423d62a80, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f8998, 0x6a551e0, 0xc42335b600, 0xc421b534b8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7ae49937f) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212e7a40, 0xc421b534a0, 0x6a551e0, 0xc42335b600, 0xc400000008, 0xc424909000, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212e7a40, 0x6a551e0, 0xc42335b600, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212e7a40, 0x6a551e0, 0xc42335b600, 0xc421b533f0, 0xc423d62a20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114948 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc420159a10, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420159a10, 0x15179d571b6047d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420159a10, 0x15179d571b6047d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420159a10, 0x15179d571b6047d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423b9bfb0, 0x15179d571b6047d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423b9bf50, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423b9bf50, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423b9bf50, 0x5c4d11c, 0x10, 0xc4238f79b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423b9bf50, 0xc4225067a8, 0xc423b9bf50, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423b9bf50, 0xc4225067a8, 0x1f, 0xc42002fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4226164d0, 0xc422506790, 0x6a551e0, 0xc424038d80, 0xc400000008, 0xc422495540, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4226164d0, 0x6a551e0, 0xc424038d80, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4226164d0, 0x6a551e0, 0xc424038d80, 0xc4225064d0, 0xc42252c060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115257 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc424596c60, 0xc420e75f80, 0xc422cd6f60, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc424596c60, 0x15179d56fce4c0b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc424596c60, 0x15179d56fce4c0b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc424596c60, 0x15179d56fce4c0b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc424596c30, 0x15179d56fce4c0b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc424596bd0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc424596bd0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc424596ba0, 0x5c4d11c, 0x10, 0xc4243c12c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc424596ba0, 0xc420e75f10, 0xc424596ba0, 0xc4267ed568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc424596ba0, 0xc420e75f10, 0xc420b53500, 0xc423f80848, 0xc423a47e40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc424596ab0, 0x5c13340, 0xc420e75f10, 0x5c13340, 0xc420e75f10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc424596ab0, 0x5c13340, 0xc420e75f10, 0xc421b1d4a0, 0xc421b1d500, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc424596ab0, 0x5c13340, 0xc420e75f10, 0xc421b1d4a0, 0xc421b1d500, 0x6, 0xc4267ed968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc424596ab0, 0xc420e75e30, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc420dc5c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423875e00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423875e00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423875e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114820 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42248f920, 0x15179d57178c29a8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42248f920, 0x15179d57178c29a8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42248f920, 0x15179d57178c29a8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42248f8c0, 0x15179d57178c29a8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42248f860, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42248f860, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42248f830, 0x5c4d11c, 0x10, 0xc4247bf2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42248f830, 0xc421a684d0, 0xc42248f830, 0xc424cdf568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42248f830, 0xc421a684d0, 0xc420b53500, 0xc423f803b8, 0xc42141cf40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4227065d0, 0x5c13340, 0xc421a684d0, 0x5c13340, 0xc421a684d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4227065d0, 0x5c13340, 0xc421a684d0, 0xc420396820, 0xc420396840, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4227065d0, 0x5c13340, 0xc421a684d0, 0xc420396820, 0xc420396840, 0x6, 0xc424cdf968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4227065d0, 0xc421a68460, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424922c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422e643c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422e643c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422e643c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115176 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4239f8d20, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4239f8d20, 0x15179d572105da38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4239f8d20, 0x15179d572105da38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4239f8d20, 0x15179d572105da38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4239f8cf0, 0x15179d572105da38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4239f8c90, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4239f8c90, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4239f8c90, 0x5c4d11c, 0x10, 0xc424b039b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4239f8c90, 0xc422c74228, 0xc4239f8c90, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4239f8c90, 0xc422c74228, 0x1f, 0xc420a82630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4210c62a0, 0xc422c74210, 0x6a551e0, 0xc4226c51c0, 0xc400000008, 0xc422280f00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4210c62a0, 0x6a551e0, 0xc4226c51c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4210c62a0, 0x6a551e0, 0xc4226c51c0, 0xc420ecd290, 0xc422971260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114645 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42377b4d0, 0xc423f2f110, 0xc424201560, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42377b4d0, 0x15179d5715df0648, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42377b4d0, 0x15179d5715df0648, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42377b4d0, 0x15179d5715df0648, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42377b440, 0x15179d5715df0648, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42377b3e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42377b3e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42377b3e0, 0x5c4d11c, 0x10, 0xc423c299b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42377b3e0, 0xc420e7ac78, 0xc42377b3e0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42377b3e0, 0xc420e7ac78, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423d9da40, 0xc420e7ac60, 0x6a551e0, 0xc424a16400, 0xc400000008, 0xc42391c380, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423d9da40, 0x6a551e0, 0xc424a16400, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423d9da40, 0x6a551e0, 0xc424a16400, 0xc420e7a210, 0xc42193fc20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115249 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc424902780, 0x45563f3, 0xc424b6f1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc424902780, 0x5d2c558, 0xc424b6f420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4219f6e70, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4219f6e70, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423e2a178, 0x5b6ec40, 0xc4237b1860, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423910940, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423e2a178, 0x5b6ec40, 0xc4237b1860, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc423910940, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423e2a178, 0x5b6ec40, 0xc4237b1860, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423910940, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423e2a178, 0x5b6ec40, 0xc4237b1860, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f8850, 0x6a551e0, 0xc423910940, 0xc423e2a178, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x77a1ed8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5d180, 0xc423e2a160, 0x6a551e0, 0xc423910940, 0xc400000008, 0xc423910a80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5d180, 0x6a551e0, 0xc423910940, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5d180, 0x6a551e0, 0xc423910940, 0xc423e2a000, 0xc4237b17a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114845 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224f3ef0, 0x15179d571a06de08, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224f3ef0, 0x15179d571a06de08, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224f3ef0, 0x15179d571a06de08, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224f3ec0, 0x15179d571a06de08, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224f3e60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224f3e60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224f3e60, 0x5c4d11c, 0x10, 0xc422e399b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224f3e60, 0xc4247c1828, 0xc4224f3e60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224f3e60, 0xc4247c1828, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420b60310, 0xc4247c1810, 0x6a551e0, 0xc4232a4480, 0xc400000008, 0xc423910780, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420b60310, 0x6a551e0, 0xc4232a4480, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420b60310, 0x6a551e0, 0xc4232a4480, 0xc4247c16b0, 0xc420efbec0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115237 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc424dc5e00, 0x45563f3, 0xc423eb91e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc424dc5e00, 0x5d2c558, 0xc423eb9420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc423008a50, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc423008a50, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474ef38, 0x5b6ec40, 0xc422226540, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42335b3c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474ef38, 0x5b6ec40, 0xc422226540, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc42335b3c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474ef38, 0x5b6ec40, 0xc422226540, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42335b3c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474ef38, 0x5b6ec40, 0xc422226540, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f86c8, 0x6a551e0, 0xc42335b3c0, 0xc42474ef38, 0x0, 0x0, 0x0, 0x1, 0x5a16d20, 0xc42138e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421037c70, 0xc42474ef20, 0x6a551e0, 0xc42335b3c0, 0xc400000008, 0xc4226c4900, 0x30928260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421037c70, 0x6a551e0, 0xc42335b3c0, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421037c70, 0x6a551e0, 0xc42335b3c0, 0xc42474ee70, 0xc4222264e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114874 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423b9be90, 0xc4206a4230, 0x7, 0xc4234e6af0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423b9be90, 0x7, 0xc4234e6af0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423b9be90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423b9be90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423b9be90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423b9be90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422fc8700, 0x6a552a0, 0xc423b9be60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42160dd00, 0xc42362ae00, 0x0, 0xc42160de38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42362ae00, 0x1, 0xc423b9be00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4226f90e0, 0xc4212e5500, 0x5, 0x5, 0x15179d571b3330d8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42375f590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115012 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc420bd5440, 0xc4202d8b90, 0x7, 0xc421989540, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc420bd5440, 0x7, 0xc421989540, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc420bd5440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc420bd5440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc420bd5440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc420bd5440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc42293bc00, 0x6a552a0, 0xc422406090, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc42293bc00, 0x6a552a0, 0xc422406090) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc421863860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc421863860, 0xc4201e5470, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc4201357d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 112424 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4237ffce0, 0xc4206a4230, 0x7, 0xc420ef49b0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4237ffce0, 0x7, 0xc420ef49b0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4237ffce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4237ffce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4237ffce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4237ffce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421a81500, 0x6a552a0, 0xc4237ffcb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421804f78, 0xc4227ece00, 0x2, 0xc424dce320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4227ece00, 0x3, 0xc424dce300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc422d52000, 0x42, 0x42, 0x15179d56dd3af6a8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc422d52000, 0x42, 0x42, 0x15179d56dd3af6a8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42244d7d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115263 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc420d69d70, 0xc421081c70, 0xc423942660, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d69d70, 0x15179d5701e88150, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d69d70, 0x15179d5701e88150, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420d69d70, 0x15179d5701e88150, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420d69d40, 0x15179d5701e88150, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420d69ce0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420d69ce0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420d69cb0, 0x5c4d11c, 0x10, 0xc422c732c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420d69cb0, 0xc421081b90, 0xc420d69cb0, 0xc4230e1568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420d69cb0, 0xc421081b90, 0xc420b53500, 0xc421ac8480, 0xc424a5af80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420bd4210, 0x5c13340, 0xc421081b90, 0x5c13340, 0xc421081b90, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420bd4210, 0x5c13340, 0xc421081b90, 0xc422d4db40, 0xc422d4db60, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420bd4210, 0x5c13340, 0xc421081b90, 0xc422d4db40, 0xc422d4db60, 0x6, 0xc4230e1968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420bd4210, 0xc421081b20, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc420ccfc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422751b80, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422751b80, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422751b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115168 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42359c840, 0xc421080620, 0xc423cf9440, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359c840, 0x15179d56f6be4a30, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359c840, 0x15179d56f6be4a30, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42359c840, 0x15179d56f6be4a30, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42359c810, 0x15179d56f6be4a30, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42359c7b0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42359c7b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42359c780, 0x5c4d11c, 0x10, 0xc42458d2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42359c780, 0xc421080540, 0xc42359c780, 0xc4230e1568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42359c780, 0xc421080540, 0xc420b53500, 0xc421ac8338, 0xc42456f9c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc42359c660, 0x5c13340, 0xc421080540, 0x5c13340, 0xc421080540, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc42359c660, 0x5c13340, 0xc421080540, 0xc422d4c860, 0xc422d4c880, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc42359c660, 0x5c13340, 0xc421080540, 0xc422d4c860, 0xc422d4c880, 0x6, 0xc4230e1968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc42359c660, 0xc420dd3ea0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0x5b8e380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4235832c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4235832c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4235832c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 113501 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421684d50, 0x15179d56f8b08ce0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421684d50, 0x15179d56f8b08ce0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421684d50, 0x15179d56f8b08ce0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc424550cf0, 0x15179d56f8b08ce0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc424550c90, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc424550c90, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc424550c90, 0x5c4d11c, 0x10, 0xc42486b9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc424550c90, 0xc4236b2e88, 0xc424550c90, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc424550c90, 0xc4236b2e88, 0x1f, 0xc42072de30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42384d340, 0xc4236b2e70, 0x6a551e0, 0xc4236594c0, 0xc400000008, 0xc42397d140, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42384d340, 0x6a551e0, 0xc4236594c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42384d340, 0x6a551e0, 0xc4236594c0, 0xc4236b2dc0, 0xc42263fc80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115030 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42166a1e0, 0xc4206a4230, 0x7, 0xc423abd180, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42166a1e0, 0x7, 0xc423abd180, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42166a1e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42166a1e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42166a1e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42166a1e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e6b500, 0x6a552a0, 0xc42166a1b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42591cf78, 0xc42546c300, 0x2, 0xc423a46d60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42546c300, 0x3, 0xc423a46d40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424b28000, 0x38, 0x38, 0x15179d571e847300, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424b28000, 0x38, 0x38, 0x15179d571e847300, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4224e0b10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114965 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc420b225a0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420b225a0, 0x15179d571ba5b158, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420b225a0, 0x15179d571ba5b158, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420b225a0, 0x15179d571ba5b158, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420b224e0, 0x15179d571ba5b158, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420b22480, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420b22480, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420b22480, 0x5c4d11c, 0x10, 0xc42482d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420b22480, 0xc423c1b408, 0xc420b22480, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420b22480, 0xc423c1b408, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423f2e150, 0xc423c1b3f0, 0x6a551e0, 0xc423aa1900, 0xc400000008, 0xc424104580, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423f2e150, 0x6a551e0, 0xc423aa1900, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423f2e150, 0x6a551e0, 0xc423aa1900, 0xc423c1ae70, 0xc42245b980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114750 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423583540, 0x45563f3, 0xc4215b31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423583540, 0x5d2c558, 0xc4215b3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4225f99b0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4225f99b0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4222b26f8, 0x5b6ec40, 0xc421864c60, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42456e9c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4222b26f8, 0x5b6ec40, 0xc421864c60, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc42456e9c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4222b26f8, 0x5b6ec40, 0xc421864c60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42456e9c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4222b26f8, 0x5b6ec40, 0xc421864c60, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f9548, 0x6a551e0, 0xc42456e9c0, 0xc4222b26f8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x801dbb5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420dd32d0, 0xc4222b26e0, 0x6a551e0, 0xc42456e9c0, 0xc400000008, 0xc422cee4c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420dd32d0, 0x6a551e0, 0xc42456e9c0, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420dd32d0, 0x6a551e0, 0xc42456e9c0, 0xc4222b22c0, 0xc421864ba0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 110457 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42387b2c0, 0xc4206a4230, 0x7, 0xc4226724b0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42387b2c0, 0x7, 0xc4226724b0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42387b2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42387b2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42387b2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42387b2c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423072fc0, 0x6a552a0, 0xc42387b290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a55260, 0xc4200cd740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a55260, 0xc4200cd740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a55260, 0xc4200cd740, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a55260, 0xc4200cd740, 0xc422633a80, 0xc4202c1800, 0x6cae460, 0x6a55260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a55260, 0xc4200cd740, 0xc4202c1800, 0xc4200cd740, 0xc4221718e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420423b00, 0x8, 0x8, 0x15179d56b62d0101, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:625 +0x7a2 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x40, 0x40, 0x15179d56b62dadf8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:365 +0x196 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc421f537a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 8562 [IO wait]: internal/poll.runtime_pollWait(0x73eb6a8, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc421acd018, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc421acd018, 0xc421939800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc421acd000, 0xc421939800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc421acd000, 0xc421939800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206ef2c0, 0xc421939800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4239f6fa0, 0xc421939800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc422b21c80, 0x73ccbf8, 0xc4239f6fa0, 0x5, 0xc4239f6fa0, 0xc6) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc42300ca80, 0x5d2d117, 0xc42300cba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc42300ca80, 0xc422b5c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4222ee9c0, 0xc422b5c000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc421f4b720) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc421f4b720, 0xbe9de8a32f9f5090, 0x7cff89371, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc421f4b850, 0x6a46020, 0xc421f4b720, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc421f4b6c0, 0x6a551e0, 0xc4229b7900, 0xc422170330, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc420f286c0, 0x6a611e0, 0xc42300ca80, 0x0, 0x0, 0xc4230b300d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc420f286c0, 0x6a611e0, 0xc42300ca80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc422b21c50, 0x6a61a80, 0xc4239f6fa0, 0xc422378130, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4239f6fa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc4239f6fa0, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 114343 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbea50, 0x15179d570f2dd428, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbea50, 0x15179d570f2dd428, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423fbea50, 0x15179d570f2dd428, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423fbe9f0, 0x15179d570f2dd428, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423fbe990, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423fbe990, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423fbe990, 0x5c4d11c, 0x10, 0xc423aaf9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423fbe990, 0xc423b31cf8, 0xc423fbe990, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423fbe990, 0xc423b31cf8, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422666690, 0xc423b31ce0, 0x6a551e0, 0xc422495380, 0xc400000008, 0xc4222818c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422666690, 0x6a551e0, 0xc422495380, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422666690, 0x6a551e0, 0xc422495380, 0xc423b31b80, 0xc421ed5500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 21423 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc42130f650, 0xc4202d8b90, 0x7, 0xc4223e87d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc42130f650, 0x7, 0xc4223e87d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc42130f650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc42130f650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc42130f650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc42130f650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4245afc00, 0x6a552a0, 0xc42130f440, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc424a5a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420db3a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc4216836c0, 0x6a551e0, 0xc424a5a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc424a5a900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc424a5a900, 0xc4243b5878, 0xc424584a00, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc4216836c0, 0x6a551e0, 0xc424a5a900, 0xc424584a00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc422ecf3d0, 0x6a551e0, 0xc424a5a900, 0xc421ef69e0, 0x13, 0x20, 0xcb8, 0x6a651a0, 0xc42326db00, 0xc4243455c0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4219f6840, 0x6a551e0, 0xc424a5a900, 0xc422ecf440, 0xc422ecf3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4219f6840, 0xc421203528, 0xc4243b6b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42397c280, 0x5c46757, 0xc, 0x6a4c120, 0xc4219f6840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42397c280, 0x6a4c120, 0xc4219f6840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc424a5a900, 0x6a4c120, 0xc4219f6840, 0x5d28ea0, 0x0, 0x0, 0xc4219f6870, 0xc421c22260, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc424a5a900, 0xc422ecf440, 0xc422ecf3d0, 0x6a4c120, 0xc4219f6840, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc424a5a900, 0xc422ecf440, 0xc422ecf3d0, 0x6a4c120, 0xc4219f6840, 0x0, 0xc421c22260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc422ecf648, 0x6a551e0, 0xc424a5a900, 0xc422ecf440, 0xc422ecf3d0, 0x8c46c6d489d7388c, 0xc4243b7590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc422ecf000, 0x6a551e0, 0xc424a5a900, 0xc422ecf3d0, 0x1, 0x73e91b0, 0xc4236938c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc422ecf000, 0x6a551e0, 0xc424a5a900, 0x6a568e0, 0xc42363a180, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc422ecf000, 0x6a551e0, 0xc424a5a900, 0x6a568e0, 0xc42363a180, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422ecf000, 0x6a551e0, 0xc424a5a900, 0x6a568e0, 0xc42363a180, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422ecf000, 0x6a551e0, 0xc420f28cc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc420f28cc0, 0x0, 0x0, 0xc42389300d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4218a3a30, 0xc4204a4700, 0x6a551e0, 0xc420f28cc0, 0xc423635500, 0x5400, 0x15000, 0xc420724a90, 0xc4226ef030, 0xc4218a3a20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114648 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4238c72c0, 0x15179d5716114978, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4238c72c0, 0x15179d5716114978, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4238c72c0, 0x15179d5716114978, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4238c7260, 0x15179d5716114978, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4238c71d0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4238c71d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4238c71d0, 0x5c4d11c, 0x10, 0xc4239419b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4238c71d0, 0xc423cd4e88, 0xc4238c71d0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4238c71d0, 0xc423cd4e88, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423f2fa40, 0xc423cd4e70, 0x6a551e0, 0xc424a17400, 0xc400000008, 0xc42160ed00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423f2fa40, 0x6a551e0, 0xc424a17400, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423f2fa40, 0x6a551e0, 0xc424a17400, 0xc420e7bad0, 0xc423caa540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115127 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4223d5f50, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4223d5f50, 0x15179d571ec11288, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421a6d800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4223d5f50, 0x15179d571ec11288, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421a6d800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4223d5f50, 0x15179d571ec11288, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421a6d800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422426510, 0x15179d571ec11288, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421a6d800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224264b0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421a6d800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224264b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422426480, 0x5c4d11c, 0x10, 0xc4241872c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422426480, 0xc422993110, 0xc422426480, 0xc42591d568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422426480, 0xc422993110, 0xc420b53500, 0xc4204f9c78, 0xc42160ef80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc422426360, 0x5c13340, 0xc422993110, 0x5c13340, 0xc422993110, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc422426360, 0x5c13340, 0xc422993110, 0xc422eb3840, 0xc422eb38c0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc422426360, 0x5c13340, 0xc422993110, 0xc422eb3840, 0xc422eb38c0, 0x6, 0xc42591d968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc422426360, 0xc422992e70, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba3c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4225f1e00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4225f1e00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4225f1e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114371 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422f60500, 0x45563f3, 0xc4248a31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422f60500, 0x5d2c558, 0xc4248a3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc422706ba0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc422706ba0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421f812a8, 0x5b6ec40, 0xc421fc6360, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423a46380, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421f812a8, 0x5b6ec40, 0xc421fc6360, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc423a46380, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421f812a8, 0x5b6ec40, 0xc421fc6360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423a46380, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421f812a8, 0x5b6ec40, 0xc421fc6360, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca060, 0x6a551e0, 0xc423a46380, 0xc421f812a8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7ea009e10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421e9c310, 0xc421f81290, 0x6a551e0, 0xc423a46380, 0xc400000008, 0xc42160e180, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421e9c310, 0x6a551e0, 0xc423a46380, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421e9c310, 0x6a551e0, 0xc423a46380, 0xc421f81080, 0xc421fc6300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115045 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc420bd4f60, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420bd4f60, 0x15179d5705e10f98, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4225d7100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420bd4f60, 0x15179d5705e10f98, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4225d7100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420bd4f60, 0x15179d5705e10f98, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4225d7100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420bd4f00, 0x15179d5705e10f98, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4225d7100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420bd4e70, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4225d7100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420bd4e70, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420bd4e40, 0x5c4d11c, 0x10, 0xc422e0f2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420bd4e40, 0xc4204a28c0, 0xc420bd4e40, 0xc42447b568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420bd4e40, 0xc4204a28c0, 0xc420b53500, 0xc421ac9050, 0xc4231b6b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420bd4b40, 0x5c13340, 0xc4204a28c0, 0x5c13340, 0xc4204a28c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420bd4b40, 0x5c13340, 0xc4204a28c0, 0xc422697360, 0xc422697380, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420bd4b40, 0x5c13340, 0xc4204a28c0, 0xc422697360, 0xc422697380, 0x6, 0xc42447b968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420bd4b40, 0xc4204a2690, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba2c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4221e1e00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4221e1e00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4221e1e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114345 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4204e0f90, 0xc4226672d0, 0xc4244dd620, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4204e0f90, 0x15179d570f588da8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4204e0f90, 0x15179d570f588da8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4204e0f90, 0x15179d570f588da8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4204e0f60, 0x15179d570f588da8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4204e0ea0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4204e0ea0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4204e0ea0, 0x5c4d11c, 0x10, 0xc42141b9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4204e0ea0, 0xc422457148, 0xc4204e0ea0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4204e0ea0, 0xc422457148, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4226671f0, 0xc422457130, 0x6a551e0, 0xc422495b00, 0xc400000008, 0xc424918800, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4226671f0, 0x6a551e0, 0xc422495b00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4226671f0, 0x6a551e0, 0xc422495b00, 0xc422456dc0, 0xc4244dd560) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115140 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4223d5f20, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4223d5f20, 0x15179d571f017eb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4223d5f20, 0x15179d571f017eb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4223d5f20, 0x15179d571f017eb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4223d5ef0, 0x15179d571f017eb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4223d5e90, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4223d5e90, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4223d5e90, 0x5c4d11c, 0x10, 0xc4240239b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4223d5e90, 0xc423a9bcf8, 0xc4223d5e90, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4223d5e90, 0xc423a9bcf8, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420cebb90, 0xc423a9bce0, 0x6a551e0, 0xc424e86480, 0xc400000008, 0xc42439cc00, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420cebb90, 0x6a551e0, 0xc424e86480, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420cebb90, 0x6a551e0, 0xc424e86480, 0xc423a9bc30, 0xc423d63440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115192 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4218bc4e0, 0xc423f2ff80, 0xc422b68060, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bc4e0, 0x15179d5721acd068, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bc4e0, 0x15179d5721acd068, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4218bc4e0, 0x15179d5721acd068, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4218bc480, 0x15179d5721acd068, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4218bc420, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4218bc420, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4218bc420, 0x5c4d11c, 0x10, 0xc422f399b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4218bc420, 0xc42371ed28, 0xc4218bc420, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4218bc420, 0xc42371ed28, 0x1f, 0xc420bcc630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423f2ff10, 0xc42371ed10, 0x6a551e0, 0xc42403ab40, 0xc400000008, 0xc4226c5dc0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423f2ff10, 0x6a551e0, 0xc42403ab40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423f2ff10, 0x6a551e0, 0xc42403ab40, 0xc42371ea50, 0xc422b68000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114373 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422707710, 0x15179d57115deee0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422707710, 0x15179d57115deee0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422707710, 0x15179d57115deee0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4227076e0, 0x15179d57115deee0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422707650, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422707650, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422707650, 0x5c4d11c, 0x10, 0xc42454d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422707650, 0xc421f81568, 0xc422707650, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422707650, 0xc421f81568, 0x1f, 0xc42002ee30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421e9caf0, 0xc421f81550, 0x6a551e0, 0xc423a46840, 0xc400000008, 0xc423e24740, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421e9caf0, 0x6a551e0, 0xc423a46840, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421e9caf0, 0x6a551e0, 0xc423a46840, 0xc421f81340, 0xc421fc68a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114341 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422c38640, 0x45563f3, 0xc42339b1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422c38640, 0x5d2c558, 0xc42339b420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42334da40, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42334da40, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31ae8, 0x5b6ec40, 0xc421ed46c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc422494f00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31ae8, 0x5b6ec40, 0xc421ed46c0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc422494f00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31ae8, 0x5b6ec40, 0xc421ed46c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc422494f00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31ae8, 0x5b6ec40, 0xc421ed46c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80a48, 0x6a551e0, 0xc422494f00, 0xc423b31ae8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7f7a6925f) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4226660e0, 0xc423b31ad0, 0x6a551e0, 0xc422494f00, 0xc400000008, 0xc424039f80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4226660e0, 0x6a551e0, 0xc422494f00, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4226660e0, 0x6a551e0, 0xc422494f00, 0xc423b31970, 0xc421ed4660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 111687 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42122c150, 0xc4206a4230, 0x7, 0xc420dacd20, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42122c150, 0x7, 0xc420dacd20, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42122c150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42122c150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42122c150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42122c150, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421f4a700, 0x6a552a0, 0xc42122c120, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424478f78, 0xc424a70700, 0x3, 0xc422ef50b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424a70700, 0x3, 0xc422ef5080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4247c2000, 0x36, 0x36, 0x15179d56cf7498a8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4247c2000, 0x36, 0x36, 0x15179d56cf7498a8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4226e4840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114625 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423890780, 0x15179d5716064cf8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423890780, 0x15179d5716064cf8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423890780, 0x15179d5716064cf8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423890720, 0x15179d5716064cf8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4238906c0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4238906c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4238906c0, 0x5c4d11c, 0x10, 0xc423ca99b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4238906c0, 0xc422108b18, 0xc4238906c0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4238906c0, 0xc422108b18, 0x1f, 0xc420a7ee30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422617a40, 0xc422108b00, 0x6a551e0, 0xc423e25980, 0xc400000008, 0xc421450b00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422617a40, 0x6a551e0, 0xc423e25980, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422617a40, 0x6a551e0, 0xc423e25980, 0xc422108790, 0xc4242a8840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 8694 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc424563290, 0xc4206a4230, 0x7, 0xc423a4e050, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc424563290, 0x7, 0xc423a4e050, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc424563290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7d500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc424563290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7d500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc424563290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7d500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc424563290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4249cf180, 0x6a552a0, 0xc424563230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4220a9a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422a7d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc420efcd00, 0x6a551e0, 0xc4220a9a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4220a9a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4220a9a80, 0xc4237a7878, 0xc422c5d500, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc420efcd00, 0x6a551e0, 0xc4220a9a80, 0xc422c5d500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc421b173d0, 0x6a551e0, 0xc4220a9a80, 0xc4218c0d20, 0x13, 0x20, 0xcd0, 0x6a651a0, 0xc42326c300, 0xc424e36f40, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc423e77860, 0x6a551e0, 0xc4220a9a80, 0xc421b17440, 0xc421b173d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc423e77860, 0xc421ba99a8, 0xc4237a8b28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42456e3c0, 0x5c46757, 0xc, 0x6a4c120, 0xc423e77860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42456e3c0, 0x6a4c120, 0xc423e77860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4220a9a80, 0x6a4c120, 0xc423e77860, 0x5d28ea0, 0x0, 0x0, 0xc423e77890, 0xc42037d180, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4220a9a80, 0xc421b17440, 0xc421b173d0, 0x6a4c120, 0xc423e77860, 0x735a6c8, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4220a9a80, 0xc421b17440, 0xc421b173d0, 0x6a4c120, 0xc423e77860, 0x0, 0xc42037d180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc421b17648, 0x6a551e0, 0xc4220a9a80, 0xc421b17440, 0xc421b173d0, 0x2402ef4a398e4a9d, 0xc4237a9590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc421b17000, 0x6a551e0, 0xc4220a9a80, 0xc421b173d0, 0x1, 0x73e91b0, 0xc422e00870, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc421b17000, 0x6a551e0, 0xc4220a9a80, 0x6a568e0, 0xc423419620, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc421b17000, 0x6a551e0, 0xc4220a9a80, 0x6a568e0, 0xc423419620, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421b17000, 0x6a551e0, 0xc4220a9a80, 0x6a568e0, 0xc423419620, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421b17000, 0x6a551e0, 0xc4229b7940, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc4229b7940, 0x0, 0x0, 0xc4230b300d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422170380, 0xc42100a460, 0x6a551e0, 0xc4229b7940, 0xc421f4b6c0, 0x5400, 0x15000, 0xc4206d5810, 0xc421edb120, 0xc422170370) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 113852 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4219aaf90, 0x15179d5701018098, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4219aaf90, 0x15179d5701018098, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4219aaf90, 0x15179d5701018098, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4219aaf60, 0x15179d5701018098, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4219aaf00, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4219aaf00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4219aaf00, 0x5c4d11c, 0x10, 0xc4237059b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4219aaf00, 0xc4208bc7a8, 0xc4219aaf00, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4219aaf00, 0xc4208bc7a8, 0x1f, 0xc420a7ee30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f703f0, 0xc4208bc790, 0x6a551e0, 0xc4241cd4c0, 0xc400000008, 0xc424dcef40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f703f0, 0x6a551e0, 0xc4241cd4c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f703f0, 0x6a551e0, 0xc4241cd4c0, 0xc423a8dce0, 0xc424ac5320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115091 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423009ec0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423009ec0, 0x15179d571c72fff0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421e2a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423009ec0, 0x15179d571c72fff0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421e2a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423009ec0, 0x15179d571c72fff0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421e2a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423009e90, 0x15179d571c72fff0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421e2a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423009e30, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421e2a300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423009e30, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423009e00, 0x5c4d11c, 0x10, 0xc4229792c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423009e00, 0xc421081420, 0xc423009e00, 0xc42447b568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423009e00, 0xc421081420, 0xc420b53500, 0xc421ac91d0, 0xc4241d27c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423009cb0, 0x5c13340, 0xc421081420, 0x5c13340, 0xc421081420, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423009cb0, 0x5c13340, 0xc421081420, 0xc421b1d820, 0xc421b1d840, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423009cb0, 0x5c13340, 0xc421081420, 0xc421b1d820, 0xc421b1d840, 0x6, 0xc42447b968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423009cb0, 0xc421081340, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba6c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fb540, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fb540, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fb540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114979 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420b26330, 0xc4206a4230, 0x7, 0xc422be7090, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420b26330, 0x7, 0xc422be7090, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420b26330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421315000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420b26330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421315000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420b26330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421315000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420b26330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120cd00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc42293b500, 0x6a552a0, 0xc4228d1350, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc42293b500, 0x6a552a0, 0xc4228d1350) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc4228bb710) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc4228bb710, 0xc4210098c0, 0x29, 0x6a652e0, 0xc420b88320, 0xc4216214c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114392 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4225d4ab0, 0x15179d570f2befc8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422089700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4225d4ab0, 0x15179d570f2befc8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422089700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4225d4ab0, 0x15179d570f2befc8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422089700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4225d4a50, 0x15179d570f2befc8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422089700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4225d4990, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422089700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4225d4990, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4225d4990, 0x5c4d11c, 0x10, 0xc4231a19b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4225d4990, 0xc4228a20c8, 0xc4225d4990, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4225d4990, 0xc4228a20c8, 0x1f, 0xc420892630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422993570, 0xc4228a20b0, 0x6a551e0, 0xc424265c00, 0xc400000008, 0xc42141dec0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422993570, 0x6a551e0, 0xc424265c00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422993570, 0x6a551e0, 0xc424265c00, 0xc4228a2000, 0xc423ad62a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114508 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422ed28a0, 0xc4206a4230, 0x7, 0xc4228f1b80, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422ed28a0, 0x7, 0xc4228f1b80, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422ed28a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422ed28a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422ed28a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422ed28a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2d340, 0x6a552a0, 0xc422ed2870, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424873d00, 0xc421b48300, 0x0, 0xc424873e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421b48300, 0x1, 0xc422ed2810) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423dba480, 0xc420c86c00, 0x5, 0x5, 0x15179d5713266658, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4219f7830) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115166 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423582c80, 0x45563f3, 0xc4216e31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423582c80, 0x5d2c558, 0xc4216e3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc421181da0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc421181da0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79568, 0x5b6ec40, 0xc423cf89c0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42456f100, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79568, 0x5b6ec40, 0xc423cf89c0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc42456f100, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79568, 0x5b6ec40, 0xc423cf89c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42456f100, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79568, 0x5b6ec40, 0xc423cf89c0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac82d0, 0x6a551e0, 0xc42456f100, 0xc422a79568, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x937c330) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420dd3b90, 0xc422a79550, 0x6a551e0, 0xc42456f100, 0xc400000008, 0xc42378d4c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420dd3b90, 0x6a551e0, 0xc42456f100, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420dd3b90, 0x6a551e0, 0xc42456f100, 0xc422a794a0, 0xc423cf8960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114319 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc42124cfc0, 0xc4202d8b90, 0x7, 0xc423729db0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc42124cfc0, 0x7, 0xc423729db0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc42124cfc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420f2df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc42124cfc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420f2df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc42124cfc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420f2df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc42124cfc0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200aeb00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc422e6ba40, 0x6a552a0, 0xc4213a5470, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc422e6ba40, 0x6a552a0, 0xc4213a5470) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc4213a4d50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc4213a4d50, 0xc4212c7020, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc42048ce10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 115028 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422dac270, 0xc4206a4230, 0x7, 0xc42269c000, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422dac270, 0x7, 0xc42269c000, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422dac270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422dac270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422dac270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422dac270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c27180, 0x6a552a0, 0xc422dac240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4224b1d00, 0xc422c59100, 0x0, 0xc4224b1e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc422c59100, 0x1, 0xc422dac1b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42301e6c0, 0xc420e52f00, 0x5, 0x5, 0x15179d571d18f068, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420733ce0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114729 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422b526f0, 0xc4206a4230, 0x7, 0xc4230860f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422b526f0, 0x7, 0xc4230860f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422b526f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422b526f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422b526f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422b526f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4230721c0, 0x6a552a0, 0xc422b526c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42185bd00, 0xc4245fae00, 0x0, 0xc42185be38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4245fae00, 0x1, 0xc422b52660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4218565a0, 0xc421baa600, 0x5, 0x5, 0x15179d5716af46c8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4224f3b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114439 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc421685e90, 0xc4229921c0, 0xc4244dc6c0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421685e90, 0x15179d571183a720, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421685e90, 0x15179d571183a720, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421685e90, 0x15179d571183a720, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421685e60, 0x15179d571183a720, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421685e00, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421685e00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421685e00, 0x5c4d11c, 0x10, 0xc4222599b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421685e00, 0xc423f5ebc8, 0xc421685e00, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421685e00, 0xc423f5ebc8, 0x1f, 0xc420bcb630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422992150, 0xc423f5ebb0, 0x6a551e0, 0xc423ef9b40, 0xc400000008, 0xc422ef4140, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422992150, 0x6a551e0, 0xc423ef9b40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422992150, 0x6a551e0, 0xc423ef9b40, 0xc423f5eb00, 0xc4244dc480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 62788 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc4228bb3e0, 0xc4202d8b90, 0x7, 0xc4247820f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc4228bb3e0, 0x7, 0xc4247820f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc4228bb3e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310d000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc4228bb3e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310d000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc4228bb3e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310d000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc4228bb3e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310cf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42117cc40, 0x6a552a0, 0xc4228bb380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310cf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc424908b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42310cf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc420c3fd40, 0x6a551e0, 0xc424908b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc420c3fd40, 0x6a551e0, 0xc424908b40, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc420c3fd40, 0x6a551e0, 0xc424908b40, 0xc420c3fd40, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc420c3fd40, 0x6a551e0, 0xc424908b40, 0xc424c27bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc424346000, 0x6a551e0, 0xc424908b40, 0x6a568e0, 0xc4212a4b40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc424346000, 0x6a551e0, 0xc424908b40, 0x6a568e0, 0xc4212a4b40, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc424346000, 0x6a551e0, 0xc42439cac0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc42439cac0, 0x0, 0x0, 0xc423bcb04a, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422b05170, 0xc4204a4700, 0x6a551e0, 0xc42439cac0, 0xc4249cea80, 0x5400, 0x15000, 0xc420724a90, 0xc4231573a0, 0xc422b05160) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114591 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42377a0c0, 0x15179d57152e2120, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422410e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42377a0c0, 0x15179d57152e2120, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422410e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42377a0c0, 0x15179d57152e2120, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422410e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42377a090, 0x15179d57152e2120, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422410e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42377a030, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422410e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42377a030, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42377a000, 0x5c4d11c, 0x10, 0xc4239b52c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42377a000, 0xc423f2e4d0, 0xc42377a000, 0xc424be3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42377a000, 0xc423f2e4d0, 0xc420b53500, 0xc421ac8b58, 0xc422cefc80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc42086fef0, 0x5c13340, 0xc423f2e4d0, 0x5c13340, 0xc423f2e4d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc42086fef0, 0x5c13340, 0xc423f2e4d0, 0xc422c0d500, 0xc422c0d520, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc42086fef0, 0x5c13340, 0xc423f2e4d0, 0xc422c0d500, 0xc422c0d520, 0x6, 0xc424be3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc42086fef0, 0xc423f2e460, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc42289bc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420421b80, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420421b80, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420421b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114954 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4219f7530, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4219f7530, 0x15179d571cf99128, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4219f7530, 0x15179d571cf99128, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4219f7530, 0x15179d571cf99128, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4219f74d0, 0x15179d571cf99128, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4219f7470, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4219f7470, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4219f7470, 0x5c4d11c, 0x10, 0xc422ad99b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4219f7470, 0xc420c74d28, 0xc4219f7470, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4219f7470, 0xc420c74d28, 0x1f, 0xc420899e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4237477a0, 0xc420c74d10, 0x6a551e0, 0xc42397d080, 0xc400000008, 0xc424a16ec0, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4237477a0, 0x6a551e0, 0xc42397d080, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4237477a0, 0x6a551e0, 0xc42397d080, 0xc420c749a0, 0xc4238eea20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 112598 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421065a40, 0xc4206a4230, 0x7, 0xc4210fdae0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421065a40, 0x7, 0xc4210fdae0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421065a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421065a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421065a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421065a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421f4afc0, 0x6a552a0, 0xc4210659e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc420a2cf78, 0xc423de3100, 0x1, 0xc4202c4d70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423de3100, 0x2, 0xc4202c4d60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421f90000, 0x44, 0x44, 0x15179d56dfcba7f0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421f90000, 0x44, 0x44, 0x15179d56dfcba7f0, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42359cc90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114481 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc420bd5800, 0xc42311cf50, 0xc423c5ce40, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420bd5800, 0x15179d57132a7568, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420bd5800, 0x15179d57132a7568, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420bd5800, 0x15179d57132a7568, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420bd57a0, 0x15179d57132a7568, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420bd5740, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420bd5740, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420bd5740, 0x5c4d11c, 0x10, 0xc423dcb9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420bd5740, 0xc4231d2dd8, 0xc420bd5740, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420bd5740, 0xc4231d2dd8, 0x1f, 0xc420bcb630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42311cee0, 0xc4231d2dc0, 0x6a551e0, 0xc4231b7700, 0xc400000008, 0xc422e31d00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42311cee0, 0x6a551e0, 0xc4231b7700, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42311cee0, 0x6a551e0, 0xc4231b7700, 0xc4231d2b00, 0xc423c5cde0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114741 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc421cb2780, 0x45563f3, 0xc423bf91e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc421cb2780, 0x5d2c558, 0xc423bf9420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc421649e60, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc421649e60, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a8cfe8, 0x5b6ec40, 0xc420efa0c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4220a9900, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a8cfe8, 0x5b6ec40, 0xc420efa0c0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc4220a9900, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a8cfe8, 0x5b6ec40, 0xc420efa0c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4220a9900, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a8cfe8, 0x5b6ec40, 0xc420efa0c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f93a0, 0x6a551e0, 0xc4220a9900, 0xc423a8cfe8, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x77daf60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4225a4a80, 0xc423a8cfd0, 0x6a551e0, 0xc4220a9900, 0xc400000008, 0xc4249087c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4225a4a80, 0x6a551e0, 0xc4220a9900, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4225a4a80, 0x6a551e0, 0xc4220a9900, 0xc423a8cdc0, 0xc420efa060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115083 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc42334cc30, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42334cc30, 0x15179d571d9d04e8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6ac00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42334cc30, 0x15179d571d9d04e8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6ac00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42334cc30, 0x15179d571d9d04e8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6ac00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42334cc00, 0x15179d571d9d04e8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6ac00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42334cba0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f6ac00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42334cba0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42334cba0, 0x5c4d11c, 0x10, 0xc423bdd9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42334cba0, 0xc42371ee88, 0xc42334cba0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42334cba0, 0xc42371ee88, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421036a80, 0xc42371ee70, 0x6a551e0, 0xc422495a00, 0xc400000008, 0xc424e86340, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421036a80, 0x6a551e0, 0xc422495a00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421036a80, 0x6a551e0, 0xc422495a00, 0xc42371edc0, 0xc42363b2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113727 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4237b7b30, 0x15179d56fb9d96c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4237b7b30, 0x15179d56fb9d96c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4237b7b30, 0x15179d56fb9d96c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4237b7aa0, 0x15179d56fb9d96c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4237b7a40, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4237b7a40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4237b7a40, 0x5c4d11c, 0x10, 0xc4245d59b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4237b7a40, 0xc423c1b4b8, 0xc4237b7a40, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4237b7a40, 0xc423c1b4b8, 0x1f, 0xc420895630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420176f50, 0xc423c1b4a0, 0x6a551e0, 0xc424a16480, 0xc400000008, 0xc4238bf900, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420176f50, 0x6a551e0, 0xc424a16480, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420176f50, 0x6a551e0, 0xc424a16480, 0xc423c1b3f0, 0xc4227e84e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115096 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422753110, 0xc4206a4230, 0x7, 0xc420ef56d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422753110, 0x7, 0xc420ef56d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422753110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422753110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422753110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422753110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4224d5340, 0x6a552a0, 0xc4227530e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc425101d00, 0xc42484a000, 0x0, 0xc425101e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42484a000, 0x1, 0xc422753080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421c34120, 0xc424337800, 0x5, 0x5, 0x15179d571ef902d8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4219abfb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115167 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42359c390, 0xc420dd3e30, 0xc423cf8e40, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359c390, 0x15179d56f6ff9508, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359c390, 0x15179d56f6ff9508, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42359c390, 0x15179d56f6ff9508, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42359c360, 0x15179d56f6ff9508, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42359c300, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420720c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42359c300, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42359c300, 0x5c4d11c, 0x10, 0xc42518d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42359c300, 0xc422a79828, 0xc42359c300, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42359c300, 0xc422a79828, 0x1f, 0xc420a7fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420dd3dc0, 0xc422a79810, 0x6a551e0, 0xc42456f440, 0xc400000008, 0xc42456f480, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420dd3dc0, 0x6a551e0, 0xc42456f440, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420dd3dc0, 0x6a551e0, 0xc42456f440, 0xc422a79760, 0xc423cf8d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114593 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4238c7110, 0xc4206a4230, 0x7, 0xc4211b9c20, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4238c7110, 0x7, 0xc4211b9c20, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4238c7110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4238c7110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4238c7110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4238c7110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e07dc0, 0x6a552a0, 0xc4238c70e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421aabd00, 0xc4221be300, 0x0, 0xc421aabe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4221be300, 0x1, 0xc4238c7080) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424b96d80, 0xc420b6fb00, 0x5, 0x5, 0x15179d5715ca1300, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42377a300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115299 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4221e0780, 0x45563f3, 0xc421ba51e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4221e0780, 0x5d2c558, 0xc421ba5420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420bd5500, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420bd5500, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a598, 0x5b6ec40, 0xc42390ade0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc422e30480, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a598, 0x5b6ec40, 0xc42390ade0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc422e30480, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a598, 0x5b6ec40, 0xc42390ade0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc422e30480, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a598, 0x5b6ec40, 0xc42390ade0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80918, 0x6a551e0, 0xc422e30480, 0xc423a9a598, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x800c49042) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422528700, 0xc423a9a580, 0x6a551e0, 0xc422e30480, 0xc400000008, 0xc422e30540, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422528700, 0x6a551e0, 0xc422e30480, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422528700, 0x6a551e0, 0xc422e30480, 0xc423a9a4d0, 0xc42390ad20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 111858 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc424551ec0, 0xc4206a4230, 0x7, 0xc420e81680, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc424551ec0, 0x7, 0xc420e81680, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc424551ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc424551ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc424551ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc424551ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422691a40, 0x6a552a0, 0xc424551e90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424478f78, 0xc424863800, 0x4, 0xc420713940) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424863800, 0x5, 0xc420713900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x3c, 0x3c, 0x15179d56d11b4b98, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x3c, 0x3c, 0x15179d56d11b4b98, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4223d4f60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114976 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc421a95770, 0xc421247420, 0xc4235344e0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a95770, 0x15179d571bfcba98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a95770, 0x15179d571bfcba98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421a95770, 0x15179d571bfcba98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421a95740, 0x15179d571bfcba98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421a95590, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421a95590, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421a95590, 0x5c4d11c, 0x10, 0xc4247499b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421a95590, 0xc422f12858, 0xc421a95590, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421a95590, 0xc422f12858, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421247260, 0xc422f12840, 0x6a551e0, 0xc423911680, 0xc400000008, 0xc42403af40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421247260, 0x6a551e0, 0xc423911680, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421247260, 0x6a551e0, 0xc423911680, 0xc422f12790, 0xc4235343c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114995 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc424597b00, 0xc4206a4230, 0x7, 0xc421b54d20, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc424597b00, 0x7, 0xc421b54d20, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc424597b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc424597b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc424597b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc424597b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4245ae700, 0x6a552a0, 0xc424597aa0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc422e0fd00, 0xc424585800, 0x0, 0xc422e0fe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424585800, 0x1, 0xc424597a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc422f7dc20, 0xc4223ac300, 0x5, 0x5, 0x15179d571d1eb110, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4211e1e90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114456 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bc270, 0x15179d571104f970, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bc270, 0x15179d571104f970, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4218bc270, 0x15179d571104f970, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4218bc240, 0x15179d571104f970, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4218bc1e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4217e2b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4218bc1e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4218bc1b0, 0x5c4d11c, 0x10, 0xc4221f92c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4218bc1b0, 0xc422306bd0, 0xc4218bc1b0, 0xc424a0d568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4218bc1b0, 0xc422306bd0, 0xc420b53500, 0xc421ac83a0, 0xc42160f940) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4218bc090, 0x5c13340, 0xc422306bd0, 0x5c13340, 0xc422306bd0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4218bc090, 0x5c13340, 0xc422306bd0, 0xc4213799c0, 0xc4213799e0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4218bc090, 0x5c13340, 0xc422306bd0, 0xc4213799c0, 0xc4213799e0, 0x6, 0xc424a0d968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4218bc090, 0xc422306b60, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc4206cfc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42348ca00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42348ca00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42348ca00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115238 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc420fb1050, 0xc420e758f0, 0xc420c71f80, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420fb1050, 0x15179d56f5fb2140, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2a100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420fb1050, 0x15179d56f5fb2140, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2a100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420fb1050, 0x15179d56f5fb2140, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2a100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420fb1020, 0x15179d56f5fb2140, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2a100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420fb0fc0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2a100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420fb0fc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420fb0e70, 0x5c4d11c, 0x10, 0xc423e8b2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420fb0e70, 0xc420e75880, 0xc420fb0e70, 0xc4267ed568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420fb0e70, 0xc420e75880, 0xc420b53500, 0xc423f807c0, 0xc423a476c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423008e10, 0x5c13340, 0xc420e75880, 0x5c13340, 0xc420e75880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423008e10, 0x5c13340, 0xc420e75880, 0xc421b1ca40, 0xc421b1ca60, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423008e10, 0x5c13340, 0xc420e75880, 0xc421b1ca40, 0xc421b1ca60, 0x6, 0xc4267ed968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423008e10, 0xc420e75810, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424925c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8140, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8140, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115260 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422751180, 0x45563f3, 0xc423c6d1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422751180, 0x5d2c558, 0xc423c6d420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc424597680, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc424597680, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31b98, 0x5b6ec40, 0xc422cd7e00, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4231b6b80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31b98, 0x5b6ec40, 0xc422cd7e00, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc4231b6b80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31b98, 0x5b6ec40, 0xc422cd7e00, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4231b6b80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31b98, 0x5b6ec40, 0xc422cd7e00, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f808b8, 0x6a551e0, 0xc4231b6b80, 0xc423b31b98, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x80d215e43) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4200df810, 0xc423b31b80, 0x6a551e0, 0xc4231b6b80, 0xc400000008, 0xc4231b6e00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4200df810, 0x6a551e0, 0xc4231b6b80, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4200df810, 0x6a551e0, 0xc4231b6b80, 0xc423b31970, 0xc422cd7c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114798 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42226cde0, 0x15179d5718ce7960, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42226cde0, 0x15179d5718ce7960, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42226cde0, 0x15179d5718ce7960, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42226cd80, 0x15179d5718ce7960, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42226cd20, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42226cd20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42226cd20, 0x5c4d11c, 0x10, 0xc4228db9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42226cd20, 0xc424709e58, 0xc42226cd20, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42226cd20, 0xc424709e58, 0x1f, 0xc42002fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f704d0, 0xc424709e40, 0x6a551e0, 0xc42141c140, 0xc400000008, 0xc423910640, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f704d0, 0x6a551e0, 0xc42141c140, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f704d0, 0x6a551e0, 0xc42141c140, 0xc424709ce0, 0xc422717980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114284 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc4230089c0, 0xc420518e60, 0x7, 0xc4224f03c0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc4230089c0, 0x7, 0xc4224f03c0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc4230089c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200ae900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc4230089c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200ae900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc4230089c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200ae900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc4230089c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200ae700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc423d31dc0, 0x6a552a0, 0xc422ed33e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc423d31dc0, 0x6a552a0, 0xc422ed33e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc422ed30e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc422ed30e0, 0xc420451650, 0x29, 0x6a652e0, 0xc420358700, 0xc4203b3a90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114667 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42387ab70, 0xc4206a4230, 0x7, 0xc420c58820, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42387ab70, 0x7, 0xc420c58820, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42387ab70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42387ab70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42387ab70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42387ab70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423d30540, 0x6a552a0, 0xc42387ab40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42591cf78, 0xc420d5bc00, 0x2, 0xc423315460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc420d5bc00, 0x3, 0xc423315440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc425186000, 0x3c, 0x3c, 0x15179d5716a69438, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc425186000, 0x3c, 0x3c, 0x15179d5716a69438, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42387aab0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114469 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b4ea80, 0x15179d56f8bce8f0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b4ea80, 0x15179d56f8bce8f0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423b4ea80, 0x15179d56f8bce8f0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423b4ea50, 0x15179d56f8bce8f0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423b4e9f0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440ce00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423b4e9f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423b4e9c0, 0x5c4d11c, 0x10, 0xc4241972c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423b4e9c0, 0xc4212e7c00, 0xc423b4e9c0, 0xc424cdf568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423b4e9c0, 0xc4212e7c00, 0xc420b53500, 0xc4204f89a0, 0xc42335b800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423b4e8a0, 0x5c13340, 0xc4212e7c00, 0x5c13340, 0xc4212e7c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423b4e8a0, 0x5c13340, 0xc4212e7c00, 0xc421fa9c80, 0xc421fa9ca0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423b4e8a0, 0x5c13340, 0xc4212e7c00, 0xc421fa9c80, 0xc421fa9ca0, 0x6, 0xc424cdf968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423b4e8a0, 0xc4212e7b90, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422803c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42259fa40, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42259fa40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42259fa40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 4982 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423fbf8f0, 0xc4206a4230, 0x7, 0xc4241ef770, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423fbf8f0, 0x7, 0xc4241ef770, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423fbf8f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423fbf8f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423fbf8f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423fbf8f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8c500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e09880, 0x6a552a0, 0xc423fbf890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8c500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc42160fd40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422e8c500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc42386fba0, 0x6a551e0, 0xc42160fd40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc42160fd40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc42160fd40, 0xc423e3b878, 0xc423ebbc00, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc42386fba0, 0x6a551e0, 0xc42160fd40, 0xc423ebbc00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc421e253d0, 0x6a551e0, 0xc42160fd40, 0xc4229e18a0, 0x13, 0x20, 0xcb6, 0x6a651a0, 0xc425486900, 0xc424a1a640, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc421117c50, 0x6a551e0, 0xc42160fd40, 0xc421e25440, 0xc421e253d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc421117c50, 0xc422715228, 0xc423e3cb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc42335ad80, 0x5c46757, 0xc, 0x6a4c120, 0xc421117c50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc42335ad80, 0x6a4c120, 0xc421117c50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc42160fd40, 0x6a4c120, 0xc421117c50, 0x5d28ea0, 0x0, 0x0, 0xc421117c80, 0xc421fa8aa0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc42160fd40, 0xc421e25440, 0xc421e253d0, 0x6a4c120, 0xc421117c50, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc42160fd40, 0xc421e25440, 0xc421e253d0, 0x6a4c120, 0xc421117c50, 0x0, 0xc421fa8aa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc421e25648, 0x6a551e0, 0xc42160fd40, 0xc421e25440, 0xc421e253d0, 0xd001fe01e52002ef, 0xc423e3d590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc421e25000, 0x6a551e0, 0xc42160fd40, 0xc421e253d0, 0x1, 0x73e91b0, 0xc421874750, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc421e25000, 0x6a551e0, 0xc42160fd40, 0x6a568e0, 0xc422309e60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc421e25000, 0x6a551e0, 0xc42160fd40, 0x6a568e0, 0xc422309e60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc421e25000, 0x6a551e0, 0xc42160fd40, 0x6a568e0, 0xc422309e60, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc421e25000, 0x6a551e0, 0xc421f17c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc421f17c00, 0x0, 0x0, 0xc421bf0022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4212c9810, 0xc42100a460, 0x6a551e0, 0xc421f17c00, 0xc422e07500, 0x5400, 0x15000, 0xc4206d5810, 0xc4225938f0, 0xc4212c9800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115169 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423642640, 0x45563f3, 0xc4248491e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423642640, 0x5d2c558, 0xc424849420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42359cd20, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42359cd20, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79988, 0x5b6ec40, 0xc423cf9da0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42456fb80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79988, 0x5b6ec40, 0xc423cf9da0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc42456fb80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79988, 0x5b6ec40, 0xc423cf9da0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42456fb80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79988, 0x5b6ec40, 0xc423cf9da0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac83a0, 0x6a551e0, 0xc42456fb80, 0xc422a79988, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x770fb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4210809a0, 0xc422a79970, 0x6a551e0, 0xc42456fb80, 0xc400000008, 0xc42456fc00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4210809a0, 0x6a551e0, 0xc42456fb80, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4210809a0, 0x6a551e0, 0xc42456fb80, 0xc422a798c0, 0xc423cf9ce0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114140 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42387b350, 0xc4206a4230, 0x7, 0xc424198eb0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42387b350, 0x7, 0xc424198eb0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42387b350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42387b350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42387b350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42387b350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ee5c00, 0x6a552a0, 0xc42387b320, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423230f78, 0xc4236dc000, 0x2, 0xc424a16620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4236dc000, 0x3, 0xc424a16600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420308a00, 0x40, 0x40, 0x15179d570b3a04b8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420308a00, 0x40, 0x40, 0x15179d570b3a04b8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42375fef0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115148 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422752c30, 0xc4206a4230, 0x7, 0xc422b5b180, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422752c30, 0x7, 0xc422b5b180, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422752c30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422752c30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422752c30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422752c30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e08700, 0x6a552a0, 0xc422752c00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc425b0bd00, 0xc42546a700, 0x0, 0xc425b0be38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42546a700, 0x1, 0xc422752ba0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421e1b320, 0xc42326cf00, 0x5, 0x5, 0x15179d57211add20, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc424563e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 113461 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422707e60, 0xc4206a4230, 0x7, 0xc4218dc5f0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422707e60, 0x7, 0xc4218dc5f0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422707e60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422707e60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422707e60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422707e60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42420a000, 0x6a552a0, 0xc422707e30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424b39d00, 0xc42484d100, 0x0, 0xc424b39e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42484d100, 0x1, 0xc422707d70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421eb97a0, 0xc42300a900, 0x5, 0x5, 0x15179d56f74b72e8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc421116540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115269 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4237fadc0, 0x45563f3, 0xc424f551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4237fadc0, 0x5d2c558, 0xc424f55420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42124dd10, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42124dd10, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4231d3828, 0x5b6ec40, 0xc4238ee480, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4224959c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4231d3828, 0x5b6ec40, 0xc4238ee480, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc4224959c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4231d3828, 0x5b6ec40, 0xc4238ee480, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4224959c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4231d3828, 0x5b6ec40, 0xc4238ee480, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca620, 0x6a551e0, 0xc4224959c0, 0xc4231d3828, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7fc73d608) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42311cb60, 0xc4231d3810, 0x6a551e0, 0xc4224959c0, 0xc400000008, 0xc424dcf7c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42311cb60, 0x6a551e0, 0xc4224959c0, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42311cb60, 0x6a551e0, 0xc4224959c0, 0xc4231d36b0, 0xc4238ee3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114728 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224f39b0, 0x15179d56fce4c0b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f72800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224f39b0, 0x15179d56fce4c0b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f72800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224f39b0, 0x15179d56fce4c0b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f72800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224f3980, 0x15179d56fce4c0b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f72800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224f3920, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f72800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224f3920, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224f38f0, 0x5c4d11c, 0x10, 0xc4219f52c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224f38f0, 0xc420d09ce0, 0xc4224f38f0, 0xc4208af568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224f38f0, 0xc420d09ce0, 0xc420b53500, 0xc421ac8fa8, 0xc424733980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4224f37d0, 0x5c13340, 0xc420d09ce0, 0x5c13340, 0xc420d09ce0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4224f37d0, 0x5c13340, 0xc420d09ce0, 0xc4223a17c0, 0xc4223a17e0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4224f37d0, 0x5c13340, 0xc420d09ce0, 0xc4223a17c0, 0xc4223a17e0, 0x6, 0xc4208af968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4224f37d0, 0xc420d09c70, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc4210a5c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423c07e00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423c07e00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423c07e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114420 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0390, 0x15179d5710e461b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0390, 0x15179d5710e461b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224e0390, 0x15179d5710e461b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224e0360, 0x15179d5710e461b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224e0300, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224e0300, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224e0300, 0x5c4d11c, 0x10, 0xc4232ad9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224e0300, 0xc422c82908, 0xc4224e0300, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224e0300, 0xc422c82908, 0x1f, 0xc420a81630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42299f420, 0xc422c828f0, 0x6a551e0, 0xc42141cc00, 0xc400000008, 0xc422ceea80, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42299f420, 0x6a551e0, 0xc42141cc00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42299f420, 0x6a551e0, 0xc42141cc00, 0xc422c82840, 0xc42263f6e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114382 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42334cab0, 0xc4206a4230, 0x7, 0xc4218dce60, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42334cab0, 0x7, 0xc4218dce60, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42334cab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42334cab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42334cab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42334cab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4245aefc0, 0x6a552a0, 0xc42334ca80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424cdef78, 0xc422772000, 0x0, 0xc421228b10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc422772000, 0x1, 0xc421228b10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4248b8000, 0x42, 0x42, 0x15179d571270aed0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4248b8000, 0x42, 0x42, 0x15179d571270aed0, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4213a4f00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114475 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420d69ec0, 0xc4206a4230, 0x7, 0xc4228f0960, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420d69ec0, 0x7, 0xc4228f0960, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420d69ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420d69ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420d69ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420d69ec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2ca80, 0x6a552a0, 0xc420d69e90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423023d00, 0xc421b46700, 0x0, 0xc423023e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421b46700, 0x1, 0xc420d69e30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4208e7320, 0xc420c86000, 0x5, 0x5, 0x15179d5712d91d30, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4230096e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115243 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4233a92c0, 0x45563f3, 0xc4248a31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4233a92c0, 0x5d2c558, 0xc4248a3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc421a3a0c0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc421a3a0c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474f618, 0x5b6ec40, 0xc422227920, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42397c200, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474f618, 0x5b6ec40, 0xc422227920, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc42397c200, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474f618, 0x5b6ec40, 0xc422227920, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42397c200, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474f618, 0x5b6ec40, 0xc422227920, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f8790, 0x6a551e0, 0xc42397c200, 0xc42474f618, 0x0, 0x0, 0x0, 0x1, 0x5a16d20, 0xc4224e6788) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5c620, 0xc42474f600, 0x6a551e0, 0xc42397c200, 0xc400000008, 0xc42397c340, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5c620, 0x6a551e0, 0xc42397c200, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5c620, 0x6a551e0, 0xc42397c200, 0xc42474f550, 0xc422227800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 35005 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc421685aa0, 0xc4202d8b90, 0x7, 0xc423c41cc0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc421685aa0, 0x7, 0xc423c41cc0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc421685aa0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422247500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc421685aa0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422247500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc421685aa0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc422247500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc421685aa0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42209a1c0, 0x6a552a0, 0xc421685a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4204e0030, 0x6a551e0, 0xc423a475c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421a6d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc422a81ba0, 0x6a551e0, 0xc423a475c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc423a475c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc423a475c0, 0xc4249bd878, 0xc424860000, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc422a81ba0, 0x6a551e0, 0xc423a475c0, 0xc424860000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4243473d0, 0x6a551e0, 0xc423a475c0, 0xc4211be6a0, 0x13, 0x20, 0xccd, 0x6a651a0, 0xc42048a000, 0xc42216cff0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc4223d5020, 0x6a551e0, 0xc423a475c0, 0xc424347440, 0xc4243473d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc4223d5020, 0xc421ba87a8, 0xc4249beb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc424e860c0, 0x5c46757, 0xc, 0x6a4c120, 0xc4223d5020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc424e860c0, 0x6a4c120, 0xc4223d5020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc423a475c0, 0x6a4c120, 0xc4223d5020, 0x5d28ea0, 0x0, 0x0, 0xc4223d5050, 0xc424bd6fa0, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc423a475c0, 0xc424347440, 0xc4243473d0, 0x6a4c120, 0xc4223d5020, 0x735ad90, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc423a475c0, 0xc424347440, 0xc4243473d0, 0x6a4c120, 0xc4223d5020, 0x0, 0xc424bd6fa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc424347648, 0x6a551e0, 0xc423a475c0, 0xc424347440, 0xc4243473d0, 0x628c3891f08d824f, 0xc4249bf590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc424347000, 0x6a551e0, 0xc423a475c0, 0xc4243473d0, 0x1, 0x73e91b0, 0xc4243daa20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc424347000, 0x6a551e0, 0xc423a475c0, 0x6a568e0, 0xc4208017a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc424347000, 0x6a551e0, 0xc423a475c0, 0x6a568e0, 0xc4208017a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc424347000, 0x6a551e0, 0xc423a475c0, 0x6a568e0, 0xc4208017a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc424347000, 0x6a551e0, 0xc4222803c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc4204a4700, 0x6a551e0, 0xc4222803c0, 0x0, 0x0, 0xc422172022, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc421047900, 0xc4204a4700, 0x6a551e0, 0xc4222803c0, 0xc4237cf500, 0x5400, 0x15000, 0xc420724a90, 0xc4237540a0, 0xc4210478f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115262 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422751680, 0x45563f3, 0xc4247271e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422751680, 0x5d2c558, 0xc424727420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc424597dd0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc424597dd0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31f08, 0x5b6ec40, 0xc42390a300, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4231b7400, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31f08, 0x5b6ec40, 0xc42390a300, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc4231b7400, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31f08, 0x5b6ec40, 0xc42390a300, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4231b7400, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31f08, 0x5b6ec40, 0xc42390a300, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f808d8, 0x6a551e0, 0xc4231b7400, 0xc423b31f08, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7e4894a9b) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4200df9d0, 0xc423b31ef0, 0x6a551e0, 0xc4231b7400, 0xc400000008, 0xc4231b7800, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4200df9d0, 0x6a551e0, 0xc4231b7400, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4200df9d0, 0x6a551e0, 0xc4231b7400, 0xc423b31d90, 0xc42390a060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115254 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc424596090, 0xc420e75b20, 0xc422cd6540, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc424596090, 0x15179d56fb4dbd60, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2ab00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc424596090, 0x15179d56fb4dbd60, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2ab00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc424596090, 0x15179d56fb4dbd60, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2ab00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc424596060, 0x15179d56fb4dbd60, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2ab00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420fb1f80, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2ab00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420fb1f80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420fb1ef0, 0x5c4d11c, 0x10, 0xc425a7f2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420fb1ef0, 0xc420e75ab0, 0xc420fb1ef0, 0xc4267ed568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420fb1ef0, 0xc420e75ab0, 0xc420b53500, 0xc423f80808, 0xc423a47940) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420fb1dd0, 0x5c13340, 0xc420e75ab0, 0x5c13340, 0xc420e75ab0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420fb1dd0, 0x5c13340, 0xc420e75ab0, 0xc421b1cf60, 0xc421b1cf80, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420fb1dd0, 0x5c13340, 0xc420e75ab0, 0xc421b1cf60, 0xc421b1cf80, 0x6, 0xc4267ed968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420fb1dd0, 0xc420e75a40, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424925c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4238752c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4238752c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4238752c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114461 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4210655f0, 0xc4206a4230, 0x7, 0xc4237bd680, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4210655f0, 0x7, 0xc4237bd680, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4210655f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4210655f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4210655f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4210655f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ace380, 0x6a552a0, 0xc4210655c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423b99d00, 0xc422c56700, 0x0, 0xc423b99e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc422c56700, 0x1, 0xc421065500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42229ab40, 0xc42326d200, 0x5, 0x5, 0x15179d57127323b8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420c0edb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114858 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423c06dc0, 0x45563f3, 0xc4215b31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423c06dc0, 0x5d2c558, 0xc4215b3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc422b21650, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc422b21650, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421632f38, 0x5b6ec40, 0xc421beb260, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424104e00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421632f38, 0x5b6ec40, 0xc421beb260, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc424104e00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421632f38, 0x5b6ec40, 0xc421beb260, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424104e00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421632f38, 0x5b6ec40, 0xc421beb260, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca1c0, 0x6a551e0, 0xc424104e00, 0xc421632f38, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x4015826) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421280a80, 0xc421632f20, 0x6a551e0, 0xc424104e00, 0xc400000008, 0xc424a17bc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421280a80, 0x6a551e0, 0xc424104e00, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421280a80, 0x6a551e0, 0xc424104e00, 0xc421632e70, 0xc421beb140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113889 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42275b7a0, 0x15179d57023b3918, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42275b7a0, 0x15179d57023b3918, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42275b7a0, 0x15179d57023b3918, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423670fc0, 0x15179d57023b3918, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423670f60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423670f60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423670f60, 0x5c4d11c, 0x10, 0xc422b759b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423670f60, 0xc424709da8, 0xc423670f60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423670f60, 0xc424709da8, 0x1f, 0xc420a81e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423864690, 0xc424709d90, 0x6a551e0, 0xc424264500, 0xc400000008, 0xc423911a80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423864690, 0x6a551e0, 0xc424264500, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423864690, 0x6a551e0, 0xc424264500, 0xc424709ce0, 0xc421865800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115144 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4239f8bd0, 0xc4206a4230, 0x7, 0xc422be8c80, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4239f8bd0, 0x7, 0xc422be8c80, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4239f8bd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4239f8bd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4239f8bd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4239f8bd0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42420b500, 0x6a552a0, 0xc4239f8ba0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424b19d00, 0xc424b8f500, 0x0, 0xc424b19e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424b8f500, 0x1, 0xc4239f8b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421e1a120, 0xc426460000, 0x5, 0x5, 0x15179d5720c7c798, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc422b14f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115314 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4219f7680, 0xc420e5d340, 0xc4237b1c80, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4219f7680, 0x15179d570dbdaa28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4219f7680, 0x15179d570dbdaa28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4219f7680, 0x15179d570dbdaa28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4219f7650, 0x15179d570dbdaa28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4219f75c0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4219f75c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4219f7590, 0x5c4d11c, 0x10, 0xc424b772c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4219f7590, 0xc420e5d2d0, 0xc4219f7590, 0xc4230e3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4219f7590, 0xc420e5d2d0, 0xc420b53500, 0xc4204f8858, 0xc423910e40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4219f71d0, 0x5c13340, 0xc420e5d2d0, 0x5c13340, 0xc420e5d2d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4219f71d0, 0x5c13340, 0xc420e5d2d0, 0xc422697460, 0xc422697480, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4219f71d0, 0x5c13340, 0xc420e5d2d0, 0xc422697460, 0xc422697480, 0x6, 0xc4230e3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4219f71d0, 0xc420e5d260, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba6c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc424902a00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc424902a00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc424902a00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115244 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc421a3b680, 0xc420e5c850, 0xc422227d40, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a3b680, 0x15179d5709108180, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a3b680, 0x15179d5709108180, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421a3b680, 0x15179d5709108180, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421a3b650, 0x15179d5709108180, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421a3b560, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421a3b560, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421a3b530, 0x5c4d11c, 0x10, 0xc4221432c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421a3b530, 0xc420e5c770, 0xc421a3b530, 0xc4230e3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421a3b530, 0xc420e5c770, 0xc420b53500, 0xc4204f87a0, 0xc42397c6c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc421a3b440, 0x5c13340, 0xc420e5c770, 0x5c13340, 0xc420e5c770, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc421a3b440, 0x5c13340, 0xc420e5c770, 0xc422696800, 0xc422696840, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc421a3b440, 0x5c13340, 0xc420e5c770, 0xc422696800, 0xc422696840, 0x6, 0xc4230e3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc421a3b440, 0xc420e5c700, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc42289bc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4233a9540, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4233a9540, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4233a9540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114356 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423008ea0, 0xc4206a4230, 0x7, 0xc4224f0a50, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423008ea0, 0x7, 0xc4224f0a50, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423008ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200aef00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423008ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200aef00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423008ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200aef00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423008ea0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200aec00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc423e2ddc0, 0x6a552a0, 0xc423d65530, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc423e2ddc0, 0x6a552a0, 0xc423d65530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc423d64ae0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc423d64ae0, 0xc4201e66c0, 0x29, 0x6a652e0, 0xc420b88320, 0xc4215eda30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 115247 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc424902000, 0x45563f3, 0xc42421b1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc424902000, 0x5d2c558, 0xc42421b420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4219f63f0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4219f63f0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474fcf8, 0x5b6ec40, 0xc4237b1320, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423910240, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474fcf8, 0x5b6ec40, 0xc4237b1320, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc423910240, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474fcf8, 0x5b6ec40, 0xc4237b1320, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423910240, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474fcf8, 0x5b6ec40, 0xc4237b1320, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f8838, 0x6a551e0, 0xc423910240, 0xc42474fcf8, 0x0, 0x0, 0x0, 0x1, 0x5a16d20, 0xc4224e6788) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5d030, 0xc42474fce0, 0x6a551e0, 0xc423910240, 0xc400000008, 0xc423910340, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5d030, 0x6a551e0, 0xc423910240, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5d030, 0x6a551e0, 0xc423910240, 0xc42474fc30, 0xc4237b11a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114465 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4210657a0, 0x15179d57129f11d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4210657a0, 0x15179d57129f11d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4210657a0, 0x15179d57129f11d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421065770, 0x15179d57129f11d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4210656e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4210656e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4210656e0, 0x5c4d11c, 0x10, 0xc423ddb9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4210656e0, 0xc422c75358, 0xc4210656e0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4210656e0, 0xc422c75358, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f58850, 0xc422c75340, 0x6a551e0, 0xc4241d2c40, 0xc400000008, 0xc423a47140, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f58850, 0x6a551e0, 0xc4241d2c40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f58850, 0x6a551e0, 0xc4241d2c40, 0xc422c75290, 0xc420b3d740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114656 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421bf2720, 0x15179d5715d94d70, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424738800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421bf2720, 0x15179d5715d94d70, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424738800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421bf2720, 0x15179d5715d94d70, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424738800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421bf26c0, 0x15179d5715d94d70, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424738800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421bf2660, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424738800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421bf2660, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421bf2630, 0x5c4d11c, 0x10, 0xc4237e32c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421bf2630, 0xc42102d1f0, 0xc421bf2630, 0xc42552d568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421bf2630, 0xc42102d1f0, 0xc420b53500, 0xc421ac8e08, 0xc424732680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc421bf24e0, 0x5c13340, 0xc42102d1f0, 0x5c13340, 0xc42102d1f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc421bf24e0, 0x5c13340, 0xc42102d1f0, 0xc42297bc80, 0xc42297bca0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc421bf24e0, 0x5c13340, 0xc42102d1f0, 0xc42297bc80, 0xc42297bca0, 0x6, 0xc42552d968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc421bf24e0, 0xc42102d180, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc4210a5c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422c12c80, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422c12c80, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422c12c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115271 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4224e15f0, 0xc42311d7a0, 0xc4238ef380, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e15f0, 0x15179d570cc20c18, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e15f0, 0x15179d570cc20c18, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224e15f0, 0x15179d570cc20c18, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224e15c0, 0x15179d570cc20c18, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224e1560, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224e1560, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224e1530, 0x5c4d11c, 0x10, 0xc4232492c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224e1530, 0xc42311d730, 0xc4224e1530, 0xc425463568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224e1530, 0xc42311d730, 0xc420b53500, 0xc4241ca670, 0xc422a5a880) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4224e1410, 0x5c13340, 0xc42311d730, 0x5c13340, 0xc42311d730, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4224e1410, 0x5c13340, 0xc42311d730, 0xc421ebe0e0, 0xc421ebe100, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4224e1410, 0x5c13340, 0xc42311d730, 0xc421ebe0e0, 0xc421ebe100, 0x6, 0xc425463968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4224e1410, 0xc42311d340, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc421b04c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4237fb900, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4237fb900, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4237fb900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 4911 [IO wait]: internal/poll.runtime_pollWait(0x938e718, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc4222d9598, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc4222d9598, 0xc423632c00, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc4222d9580, 0xc423632c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc4222d9580, 0xc423632c00, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc423143530, 0xc423632c00, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc423be92c0, 0xc423632c00, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc4237ffe60, 0x73ccbf8, 0xc423be92c0, 0x5, 0xc423be92c0, 0xaf) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422e28a80, 0x5d2d117, 0xc422e28ba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422e28a80, 0xc4232b2000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4223c3d40, 0xc4232b2000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc422e07560) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc422e07560, 0xbe9de8a32086f720, 0x7c0e05aaa, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc422e07690, 0x6a46020, 0xc422e07560, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc422e07500, 0x6a551e0, 0xc421f17b40, 0xc4212c97e0, 0xc42100a460, 0x5400, 0x15000, 0xc4206d5810, 0xc421163e60, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc422965e80, 0x6a611e0, 0xc422e28a80, 0x0, 0x0, 0xc421bf0022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc4206d5680, 0x6a551e0, 0xc422965e80, 0x6a611e0, 0xc422e28a80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc4237ffe30, 0x6a61a80, 0xc423be92c0, 0xc4224be910, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc423be92c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc421163e60, 0x6a552a0, 0xc420e49ef0, 0xc4206ee360, 0x6a61a80, 0xc423be92c0, 0xc420e49f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 115189 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc422752db0, 0xc423f2eee0, 0xc423140ba0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422752db0, 0x15179d57216d98d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422752db0, 0x15179d57216d98d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422752db0, 0x15179d57216d98d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422752d80, 0x15179d57216d98d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422752cf0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422752cf0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422752cf0, 0x5c4d11c, 0x10, 0xc4233cb9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422752cf0, 0xc42371e7a8, 0xc422752cf0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422752cf0, 0xc42371e7a8, 0x1f, 0xc420bcbe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423f2ee70, 0xc42371e790, 0x6a551e0, 0xc42403a4c0, 0xc400000008, 0xc422281fc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423f2ee70, 0x6a551e0, 0xc42403a4c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423f2ee70, 0x6a551e0, 0xc42403a4c0, 0xc42371e4d0, 0xc423140b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115282 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423642b40, 0x45563f3, 0xc4215b31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423642b40, 0x5d2c558, 0xc4215b3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42359d7d0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42359d7d0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79ae8, 0x5b6ec40, 0xc42395c840, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424a5a000, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79ae8, 0x5b6ec40, 0xc42395c840, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424a5a000, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79ae8, 0x5b6ec40, 0xc42395c840, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424a5a000, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79ae8, 0x5b6ec40, 0xc42395c840, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac83e0, 0x6a551e0, 0xc424a5a000, 0xc422a79ae8, 0x0, 0x0, 0x0, 0x1, 0x5a16d20, 0xc42138e408) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421080d90, 0xc422a79ad0, 0x6a551e0, 0xc424a5a000, 0xc400000008, 0xc424a5a080, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421080d90, 0x6a551e0, 0xc424a5a000, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421080d90, 0x6a551e0, 0xc424a5a000, 0xc422a79a20, 0xc42395c6c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115193 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421116600, 0xc4206a4230, 0x7, 0xc4210fdc20, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421116600, 0x7, 0xc4210fdc20, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421116600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421116600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421116600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421116600, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42420bc00, 0x6a552a0, 0xc421116450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423dbfd00, 0xc423ebd100, 0x0, 0xc423dbfe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423ebd100, 0x1, 0xc4211163c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421856fc0, 0xc426461b00, 0x5, 0x5, 0x15179d5721ada358, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4218bc5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 113179 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4223d5d70, 0xc4206a4230, 0x7, 0xc4224ce140, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4223d5d70, 0x7, 0xc4224ce140, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4223d5d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4223d5d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4223d5d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4223d5d70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4236448c0, 0x6a552a0, 0xc4223d5d40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a55260, 0xc423d633e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a55260, 0xc423d633e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a55260, 0xc423d633e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a55260, 0xc423d633e0, 0xc42303da80, 0xc4225f5500, 0x6cae460, 0x6a55260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a55260, 0xc423d633e0, 0xc4225f5500, 0xc423d633e0, 0xc4212293f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420b68d80, 0x2, 0x2, 0x15179d56f49c0101, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:625 +0x7a2 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424340000, 0x34, 0x34, 0x15179d56f49c25b0, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:365 +0x196 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423685260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 4994 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420163b00, 0xc4206a4230, 0x7, 0xc420dac730, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420163b00, 0x7, 0xc420dac730, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420163b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421654e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420163b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421654e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420163b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421654e00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420163b00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421654d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2d6c0, 0x6a552a0, 0xc420163a40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421654d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a551e0, 0xc4231b6540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc421654d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423867ba0, 0x6a551e0, 0xc4231b6540, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc423867ba0, 0x6a551e0, 0xc4231b6540, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc423867ba0, 0x6a551e0, 0xc4231b6540, 0xc423867ba0, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc423867ba0, 0x6a551e0, 0xc4231b6540, 0xc422cc5bb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc420f56000, 0x6a551e0, 0xc4231b6540, 0x6a568e0, 0xc421ed5aa0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc420f56000, 0x6a551e0, 0xc4231b6540, 0x6a568e0, 0xc421ed5aa0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc420f56000, 0x6a551e0, 0xc423908cc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a460, 0x6a551e0, 0xc423908cc0, 0x0, 0x0, 0xc423735037, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc421063540, 0xc42100a460, 0x6a551e0, 0xc423908cc0, 0xc423645180, 0x5400, 0x15000, 0xc4206d5810, 0xc422329530, 0xc421063530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 115133 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc422556120, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422556120, 0x15179d571f0cec80, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421c06200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422556120, 0x15179d571f0cec80, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421c06200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422556120, 0x15179d571f0cec80, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421c06200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423720c60, 0x15179d571f0cec80, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421c06200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423720c00, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421c06200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423720c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423720c00, 0x5c4d11c, 0x10, 0xc421f5d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423720c00, 0xc42352fa38, 0xc423720c00, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423720c00, 0xc42352fa38, 0x1f, 0xc420a82630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421109880, 0xc42352fa20, 0x6a551e0, 0xc424a74080, 0xc400000008, 0xc424dcf900, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421109880, 0x6a551e0, 0xc424a74080, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421109880, 0x6a551e0, 0xc424a74080, 0xc42352f970, 0xc4244d97a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113614 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4237ff8f0, 0xc4206a4230, 0x7, 0xc4232b0320, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4237ff8f0, 0x7, 0xc4232b0320, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4237ff8f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4237ff8f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4237ff8f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4237ff8f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ee41c0, 0x6a552a0, 0xc4237ff8c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423c14f78, 0xc4221bd500, 0x1, 0xc42297b6f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4221bd500, 0x2, 0xc42297b6e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x3e, 0x3e, 0x15179d56fb88c6a8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x3e, 0x3e, 0x15179d56fb88c6a8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4207337d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114333 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4211169f0, 0x15179d570f359c58, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4211169f0, 0x15179d570f359c58, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4211169f0, 0x15179d570f359c58, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421116990, 0x15179d570f359c58, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4211168a0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4211168a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4211168a0, 0x5c4d11c, 0x10, 0xc4234359b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4211168a0, 0xc422a796c8, 0xc4211168a0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4211168a0, 0xc422a796c8, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422666770, 0xc422a796b0, 0x6a551e0, 0xc4241d3c80, 0xc400000008, 0xc4226c4100, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422666770, 0x6a551e0, 0xc4241d3c80, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422666770, 0x6a551e0, 0xc4241d3c80, 0xc422a79600, 0xc421ed59e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114576 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4237733e0, 0xc4206a4230, 0x7, 0xc4210e6f00, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4237733e0, 0x7, 0xc4210e6f00, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4237733e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4237733e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4237733e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4237733e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc424838e00, 0x6a552a0, 0xc423773350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4217f7d00, 0xc421948e00, 0x0, 0xc4217f7e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421948e00, 0x1, 0xc4237732f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4234fc360, 0xc420b6f500, 0x5, 0x5, 0x15179d57156df110, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423b44d50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 112765 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420c5bec0, 0xc4206a4230, 0x7, 0xc423080fa0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420c5bec0, 0x7, 0xc423080fa0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420c5bec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420c5bec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420c5bec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420c5bec0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421acf6c0, 0x6a552a0, 0xc420c5be90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421434f78, 0xc42216aa00, 0x5, 0xc4231fb450) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42216aa00, 0x5, 0xc4231fb400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420308a00, 0x3c, 0x3c, 0x15179d56e6d9cb08, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420308a00, 0x3c, 0x3c, 0x15179d56e6d9cb08, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc421117530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115079 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42334dd70, 0xc4206a4230, 0x7, 0xc4211e4410, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42334dd70, 0x7, 0xc4211e4410, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42334dd70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42334dd70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42334dd70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42334dd70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2c700, 0x6a552a0, 0xc42334dd40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4243a5d00, 0xc42235bc00, 0x0, 0xc4243a5e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42235bc00, 0x1, 0xc42334dc80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423dbb7a0, 0xc423944300, 0x5, 0x5, 0x15179d571ddb9c58, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423b4f710) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114247 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc421a95170, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a95170, 0x15179d570dd57bd0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a95170, 0x15179d570dd57bd0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421a95170, 0x15179d570dd57bd0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421a94900, 0x15179d570dd57bd0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421a948a0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421a948a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421a948a0, 0x5c4d11c, 0x10, 0xc4252379b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421a948a0, 0xc423e2a6f8, 0xc421a948a0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421a948a0, 0xc423e2a6f8, 0x1f, 0xc420a81630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423746310, 0xc423e2a6e0, 0x6a551e0, 0xc4226c4640, 0xc400000008, 0xc424105a00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423746310, 0x6a551e0, 0xc4226c4640, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423746310, 0x6a551e0, 0xc4226c4640, 0xc423e2a630, 0xc423e0a720) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114592 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42380a480, 0xc4206a4230, 0x7, 0xc420b05d10, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42380a480, 0x7, 0xc420b05d10, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42380a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42380a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42380a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42380a480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42341e380, 0x6a552a0, 0xc42380a450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424be2f78, 0xc42194b800, 0x0, 0xc42048cb60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42194b800, 0x1, 0xc42048cb60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424b24000, 0x38, 0x38, 0x15179d5715c87cc0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424b24000, 0x38, 0x38, 0x15179d5715c87cc0, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42377a2d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114135 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4235dadb0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4235dadb0, 0x15179d570a7f8a70, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4235dadb0, 0x15179d570a7f8a70, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4235dadb0, 0x15179d570a7f8a70, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4235dad80, 0x15179d570a7f8a70, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4235dad20, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4235dad20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4235dad20, 0x5c4d11c, 0x10, 0xc422e779b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4235dad20, 0xc421632a68, 0xc4235dad20, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4235dad20, 0xc421632a68, 0x1f, 0xc420892630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420d08ee0, 0xc421632a50, 0x6a551e0, 0xc421453340, 0xc400000008, 0xc423911580, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420d08ee0, 0x6a551e0, 0xc421453340, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420d08ee0, 0x6a551e0, 0xc421453340, 0xc4216329a0, 0xc421335380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114417 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc423d65b30, 0xc422666cb0, 0xc4244d96e0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d65b30, 0x15179d5711f8ac28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d65b30, 0x15179d5711f8ac28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423d65b30, 0x15179d5711f8ac28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423d65b00, 0x15179d5711f8ac28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423d65aa0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423d65aa0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423d65aa0, 0x5c4d11c, 0x10, 0xc42523f9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423d65aa0, 0xc42371f4b8, 0xc423d65aa0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423d65aa0, 0xc42371f4b8, 0x1f, 0xc42072de30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422666c40, 0xc42371f4a0, 0x6a551e0, 0xc424265ec0, 0xc400000008, 0xc424918600, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422666c40, 0x6a551e0, 0xc424265ec0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422666c40, 0x6a551e0, 0xc424265ec0, 0xc42371f3f0, 0xc4244d9620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114895 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4235da060, 0x15179d5701d0e658, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420bdbf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4235da060, 0x15179d5701d0e658, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420bdbf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4235da060, 0x15179d5701d0e658, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420bdbf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4235da030, 0x15179d5701d0e658, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420bdbf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423407fb0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420bdbf00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423407fb0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423407f80, 0x5c4d11c, 0x10, 0xc422fc32c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423407f80, 0xc42102c4d0, 0xc423407f80, 0xc423167568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423407f80, 0xc42102c4d0, 0xc420b53500, 0xc421ac83a8, 0xc4220a8900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423407d10, 0x5c13340, 0xc42102c4d0, 0x5c13340, 0xc42102c4d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423407d10, 0x5c13340, 0xc42102c4d0, 0xc4223a1380, 0xc4223a13a0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423407d10, 0x5c13340, 0xc42102c4d0, 0xc4223a1380, 0xc4223a13a0, 0x6, 0xc423167968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423407d10, 0xc42102c460, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424876c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42388a3c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42388a3c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42388a3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114893 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423406cc0, 0x15179d571a5553f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423406cc0, 0x15179d571a5553f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423406cc0, 0x15179d571a5553f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423406c90, 0x15179d571a5553f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423406c30, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423406c30, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423406c30, 0x5c4d11c, 0x10, 0xc42355f9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423406c30, 0xc421395f08, 0xc423406c30, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423406c30, 0xc421395f08, 0x1f, 0xc420892630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4213a7ce0, 0xc421395ef0, 0x6a551e0, 0xc424919a40, 0xc400000008, 0xc424105300, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4213a7ce0, 0x6a551e0, 0xc424919a40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4213a7ce0, 0x6a551e0, 0xc424919a40, 0xc421395550, 0xc423d744e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114846 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc424597cb0, 0xc420518e60, 0x7, 0xc4234e7590, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc424597cb0, 0x7, 0xc4234e7590, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc424597cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc424597cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc424597cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc424597cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc424b9cc40, 0x6a552a0, 0xc42370eb10, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc424b9cc40, 0x6a552a0, 0xc42370eb10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc42370e7e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc42370e7e0, 0xc422f3d230, 0x29, 0x6a652e0, 0xc420358700, 0xc422170960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 115106 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423d64e70, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d64e70, 0x15179d571e7eb640, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d64e70, 0x15179d571e7eb640, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423d64e70, 0x15179d571e7eb640, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423d64de0, 0x15179d571e7eb640, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423d64d80, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423d64d80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423d64d80, 0x5c4d11c, 0x10, 0xc4218859b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423d64d80, 0xc42371fe58, 0xc423d64d80, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423d64d80, 0xc42371fe58, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4203432d0, 0xc42371fe40, 0x6a551e0, 0xc42335aa00, 0xc400000008, 0xc423911d00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4203432d0, 0x6a551e0, 0xc42335aa00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4203432d0, 0x6a551e0, 0xc42335aa00, 0xc42371fb80, 0xc420ba5b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115241 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc423009710, 0xc420e5c230, 0xc422227260, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423009710, 0x15179d56feda37b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423009710, 0x15179d56feda37b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423009710, 0x15179d56feda37b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4230096e0, 0x15179d56feda37b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423009680, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423009680, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423009650, 0x5c4d11c, 0x10, 0xc422d092c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423009650, 0xc420e5c1c0, 0xc423009650, 0xc4230e3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423009650, 0xc420e5c1c0, 0xc420b53500, 0xc4204f8730, 0xc42335ba80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423009530, 0x5c13340, 0xc420e5c1c0, 0x5c13340, 0xc420e5c1c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423009530, 0x5c13340, 0xc420e5c1c0, 0xc422696140, 0xc422696160, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423009530, 0x5c13340, 0xc420e5c1c0, 0xc422696140, 0xc422696160, 0x6, 0xc4230e3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423009530, 0xc420e5c150, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0x5b8e380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8b40, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8b40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4233a8b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114794 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4239f9b90, 0x15179d5718a16a38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4239f9b90, 0x15179d5718a16a38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4239f9b90, 0x15179d5718a16a38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4239f9b60, 0x15179d5718a16a38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4239f9b00, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4239f9b00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4239f9b00, 0x5c4d11c, 0x10, 0xc42240f9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4239f9b00, 0xc4247094b8, 0xc4239f9b00, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4239f9b00, 0xc4247094b8, 0x1f, 0xc420892e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4210c79d0, 0xc4247094a0, 0x6a551e0, 0xc42403b840, 0xc400000008, 0xc4241051c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4210c79d0, 0x6a551e0, 0xc42403b840, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4210c79d0, 0x6a551e0, 0xc42403b840, 0xc4247093f0, 0xc422716840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115164 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4218bc360, 0xc4206a4230, 0x7, 0xc4228e6370, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4218bc360, 0x7, 0xc4228e6370, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4218bc360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4218bc360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4218bc360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4218bc360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e08fc0, 0x6a552a0, 0xc4218bc330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42385af78, 0xc42546bc00, 0x0, 0xc4203dab00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42546bc00, 0x1, 0xc4203dab00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420c08000, 0x32, 0x32, 0x15179d5721aa4be0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420c08000, 0x32, 0x32, 0x15179d5721aa4be0, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4211804b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 10952 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc423b443f0, 0xc420518e60, 0x7, 0xc42484fbd0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc423b443f0, 0x7, 0xc42484fbd0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc423b443f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42104df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc423b443f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42104df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc423b443f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc42104df00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc423b443f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423597f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e07a40, 0x6a552a0, 0xc423b9ae40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423597f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc4220a9b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc423597f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc420efd040, 0x6a551e0, 0xc4220a9b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send-fm(0x6a551e0, 0xc4220a9b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a551e0, 0xc4220a9b40, 0xc42564d878, 0xc4227ef800, 0xdb, 0xdb) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Run(0xc420efd040, 0x6a551e0, 0xc4220a9b40, 0xc4227ef800, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:505 +0xd1 github.com/cockroachdb/cockroach/pkg/sql.(*planner).createDescriptorWithID(0xc4232fb3d0, 0x6a551e0, 0xc4220a9b40, 0xc42250ef60, 0x13, 0x20, 0xcc6, 0x6a651a0, 0xc4237baf00, 0xc423156780, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/descriptor.go:144 +0x30f github.com/cockroachdb/cockroach/pkg/sql.(*createTableNode).startExec(0xc423684900, 0x6a551e0, 0xc4220a9b40, 0xc4232fb440, 0xc4232fb3d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/create_table.go:159 +0x406 github.com/cockroachdb/cockroach/pkg/sql.startExec.func2(0x5c46757, 0xc, 0x6a4c120, 0xc423684900, 0xc423847528, 0xc42564eb28) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:457 +0x90 github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit.func1(0xc4238bec40, 0x5c46757, 0xc, 0x6a4c120, 0xc423684900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:90 +0x5f github.com/cockroachdb/cockroach/pkg/sql.(*planVisitor).visit(0xc4238bec40, 0x6a4c120, 0xc423684900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:482 +0x3dd github.com/cockroachdb/cockroach/pkg/sql.walkPlan(0x6a551e0, 0xc4220a9b40, 0x6a4c120, 0xc423684900, 0x5d28ea0, 0x0, 0x0, 0xc423684930, 0xc422e8f660, 0x20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/walk.go:53 +0x135 github.com/cockroachdb/cockroach/pkg/sql.startExec(0x6a551e0, 0xc4220a9b40, 0xc4232fb440, 0xc4232fb3d0, 0x6a4c120, 0xc423684900, 0x735b458, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:462 +0xd2 github.com/cockroachdb/cockroach/pkg/sql.startPlan(0x6a551e0, 0xc4220a9b40, 0xc4232fb440, 0xc4232fb3d0, 0x6a4c120, 0xc423684900, 0x0, 0xc422e8f660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:401 +0x61 github.com/cockroachdb/cockroach/pkg/sql.(*planTop).start(0xc4232fb648, 0x6a551e0, 0xc4220a9b40, 0xc4232fb440, 0xc4232fb3d0, 0xdc5342034ee0b08e, 0xc42564f590) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/plan.go:390 +0xc0 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithLocalEngine(0xc4232fb000, 0x6a551e0, 0xc4220a9b40, 0xc4232fb3d0, 0x1, 0x73e91b0, 0xc4219c1440, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:573 +0x113 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc4232fb000, 0x6a551e0, 0xc4220a9b40, 0x6a568e0, 0xc4210a2060, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:537 +0x914 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc4232fb000, 0x6a551e0, 0xc4220a9b40, 0x6a568e0, 0xc4210a2060, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:333 +0x929 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc4232fb000, 0x6a551e0, 0xc4220a9b40, 0x6a568e0, 0xc4210a2060, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:82 +0x375 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc4232fb000, 0x6a551e0, 0xc4229b71c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc4229b71c0, 0x0, 0x0, 0xc422c1a00d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc422b76a20, 0xc42100a000, 0x6a551e0, 0xc4229b71c0, 0xc42209b880, 0x5400, 0x15000, 0xc42136a190, 0xc421f3b710, 0xc422b76a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114522 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4212ce360, 0x15179d5712e951a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4212ce360, 0x15179d5712e951a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4212ce360, 0x15179d5712e951a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4212ce330, 0x15179d5712e951a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4212ce2d0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4212ce2d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4212ce270, 0x5c4d11c, 0x10, 0xc4235c52c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4212ce270, 0xc421037f10, 0xc4212ce270, 0xc421c77568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4212ce270, 0xc421037f10, 0xc420b53500, 0xc423f80600, 0xc424a5b540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4212ce120, 0x5c13340, 0xc421037f10, 0x5c13340, 0xc421037f10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4212ce120, 0x5c13340, 0xc421037f10, 0xc422696ea0, 0xc422696ee0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4212ce120, 0x5c13340, 0xc421037f10, 0xc422696ea0, 0xc422696ee0, 0x6, 0xc421c77968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4212ce120, 0xc421037ea0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422b89c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4233a8a00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4233a8a00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4233a8a00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114599 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4228d0300, 0xc4206a4230, 0x7, 0xc4234e7f40, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4228d0300, 0x7, 0xc4234e7f40, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4228d0300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4228d0300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4228d0300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4228d0300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42222a540, 0x6a552a0, 0xc4228d02d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424be2f78, 0xc423b89100, 0x0, 0xc42164f260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423b89100, 0x1, 0xc42164f260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4231ee000, 0x38, 0x38, 0x15179d57156c89b0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4231ee000, 0x38, 0x38, 0x15179d57156c89b0, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4228d0270) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115071 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42166a420, 0xc4212e61c0, 0xc4234f3860, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42166a420, 0x15179d571eafb920, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42166a420, 0x15179d571eafb920, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42166a420, 0x15179d571eafb920, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42166a3c0, 0x15179d571eafb920, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42166a2d0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42166a2d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42166a2d0, 0x5c4d11c, 0x10, 0xc4235ad9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42166a2d0, 0xc422c75148, 0xc42166a2d0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42166a2d0, 0xc422c75148, 0x1f, 0xc420893e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212e6150, 0xc422c75130, 0x6a551e0, 0xc423a46e80, 0xc400000008, 0xc422a5a040, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212e6150, 0x6a551e0, 0xc423a46e80, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212e6150, 0x6a551e0, 0xc423a46e80, 0xc422c74e70, 0xc4234f3800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114556 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d246c0, 0x15179d571532d440, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d246c0, 0x15179d571532d440, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420d246c0, 0x15179d571532d440, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420d245d0, 0x15179d571532d440, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420d24330, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420d24330, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420d24330, 0x5c4d11c, 0x10, 0xc42191d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420d24330, 0xc423e71a38, 0xc420d24330, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420d24330, 0xc423e71a38, 0x1f, 0xc420a7c630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423747960, 0xc423e71a20, 0x6a551e0, 0xc422e31980, 0xc400000008, 0xc423307140, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423747960, 0x6a551e0, 0xc422e31980, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423747960, 0x6a551e0, 0xc422e31980, 0xc423e71600, 0xc42353bf80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115065 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423292f30, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423292f30, 0x15179d570bdb0778, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4200aff00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423292f30, 0x15179d570bdb0778, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4200aff00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423292f30, 0x15179d570bdb0778, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4200aff00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423292f00, 0x15179d570bdb0778, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4200aff00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423292ea0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4200aff00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423292ea0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423292ea0, 0x5c4d11c, 0x10, 0xc421ba59b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423292ea0, 0xc422c74178, 0xc423292ea0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423292ea0, 0xc422c74178, 0x1f, 0xc420893e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4239382a0, 0xc422c74160, 0x6a551e0, 0xc424a5be00, 0xc400000008, 0xc423aa0cc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4239382a0, 0x6a551e0, 0xc424a5be00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4239382a0, 0x6a551e0, 0xc424a5be00, 0xc421b53ef0, 0xc4234194a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115074 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4210657d0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4210657d0, 0x15179d571db9dfa0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4210657d0, 0x15179d571db9dfa0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4210657d0, 0x15179d571db9dfa0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421065740, 0x15179d571db9dfa0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421065650, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421065650, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421065650, 0x5c4d11c, 0x10, 0xc4242759b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421065650, 0xc422a79b98, 0xc421065650, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421065650, 0xc422a79b98, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e74070, 0xc422a79b80, 0x6a551e0, 0xc422494780, 0xc400000008, 0xc421450ac0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e74070, 0x6a551e0, 0xc422494780, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e74070, 0x6a551e0, 0xc422494780, 0xc422a79ad0, 0xc423c5c2a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114978 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422fad900, 0x45563f3, 0xc424cf31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422fad900, 0x5d2c558, 0xc424cf3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4228bb530, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4228bb530, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422b64a68, 0x5b6ec40, 0xc4228f2ba0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc422ceef80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422b64a68, 0x5b6ec40, 0xc4228f2ba0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc422ceef80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422b64a68, 0x5b6ec40, 0xc4228f2ba0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc422ceef80, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422b64a68, 0x5b6ec40, 0xc4228f2ba0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac8828, 0x6a551e0, 0xc422ceef80, 0xc422b64a68, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x9364f88) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420be4000, 0xc422b64a50, 0x6a551e0, 0xc422ceef80, 0xc400000008, 0xc4241cc740, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420be4000, 0x6a551e0, 0xc422ceef80, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420be4000, 0x6a551e0, 0xc422ceef80, 0xc422b648f0, 0xc4228f2b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113848 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4219aade0, 0xc4206a4230, 0x7, 0xc4233d4be0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4219aade0, 0x7, 0xc4233d4be0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4219aade0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4219aade0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4219aade0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4219aade0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423073180, 0x6a552a0, 0xc4219aadb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4229afd00, 0xc42484b500, 0x0, 0xc4229afe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42484b500, 0x1, 0xc4219aad20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42399ca20, 0xc423e94600, 0x5, 0x5, 0x15179d5700be5548, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423f60b10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 112309 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421a3a8d0, 0xc4206a4230, 0x7, 0xc4218ddd60, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421a3a8d0, 0x7, 0xc4218ddd60, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421a3a8d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421a3a8d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421a3a8d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421a3a8d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c26a80, 0x6a552a0, 0xc421a3a8a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a55260, 0xc4238ef200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a55260, 0xc4238ef200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a55260, 0xc4238ef200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a55260, 0xc4238ef200, 0xc42249fa80, 0xc4236b0300, 0x6cae460, 0x6a55260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a55260, 0xc4238ef200, 0xc4236b0300, 0xc4238ef200, 0xc420047f10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc422002fc0, 0x4, 0x4, 0x15179d56db6b0101, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:625 +0x7a2 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423a70000, 0x3e, 0x3e, 0x15179d56db6bdbf8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:365 +0x196 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4222da780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115033 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422a7b2c0, 0x45563f3, 0xc424ec11e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422a7b2c0, 0x5d2c558, 0xc424ec1420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4218bccf0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4218bccf0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9b148, 0x5b6ec40, 0xc4223259e0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423e25880, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9b148, 0x5b6ec40, 0xc4223259e0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc423e25880, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9b148, 0x5b6ec40, 0xc4223259e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423e25880, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9b148, 0x5b6ec40, 0xc4223259e0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f808a0, 0x6a551e0, 0xc423e25880, 0xc423a9b148, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x807c9da0f) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4202db960, 0xc423a9b130, 0x6a551e0, 0xc423e25880, 0xc400000008, 0xc424105380, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4202db960, 0x6a551e0, 0xc423e25880, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4202db960, 0x6a551e0, 0xc423e25880, 0xc423a9b080, 0xc422325980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114774 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc422b14bd0, 0xc420dd39d0, 0xc4218656e0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b14bd0, 0x15179d57177ec3f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b14bd0, 0x15179d57177ec3f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422b14bd0, 0x15179d57177ec3f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422b14b70, 0x15179d57177ec3f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422b14b10, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422b14b10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422b14b10, 0x5c4d11c, 0x10, 0xc4219879b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422b14b10, 0xc421a65a38, 0xc422b14b10, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422b14b10, 0xc421a65a38, 0x1f, 0xc420bc7630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422759180, 0xc421a65a20, 0x6a551e0, 0xc4232a5ec0, 0xc400000008, 0xc4226c5f40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422759180, 0x6a551e0, 0xc4232a5ec0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422759180, 0x6a551e0, 0xc4232a5ec0, 0xc421a65970, 0xc420de7140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114117 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4229a0750, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4229a0750, 0x15179d5709fa02d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4229a0750, 0x15179d5709fa02d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4229a0750, 0x15179d5709fa02d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4229a06c0, 0x15179d5709fa02d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4229a0600, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4229a0600, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4229a0600, 0x5c4d11c, 0x10, 0xc4230d99b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4229a0600, 0xc423270bc8, 0xc4229a0600, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4229a0600, 0xc423270bc8, 0x1f, 0xc420897630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420b332d0, 0xc423270bb0, 0x6a551e0, 0xc424919f40, 0xc400000008, 0xc424dce3c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420b332d0, 0x6a551e0, 0xc424919f40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420b332d0, 0x6a551e0, 0xc424919f40, 0xc423270790, 0xc423cabc20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114394 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4225d5170, 0x15179d56f6be4a30, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4225d5170, 0x15179d56f6be4a30, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4225d5170, 0x15179d56f6be4a30, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42248e570, 0x15179d56f6be4a30, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42248e510, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422088100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42248e510, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42248e4e0, 0x5c4d11c, 0x10, 0xc423e492c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42248e4e0, 0xc421a680e0, 0xc42248e4e0, 0xc424691568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42248e4e0, 0xc421a680e0, 0xc420b53500, 0xc4204f8068, 0xc423ef8040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc42248e1e0, 0x5c13340, 0xc421a680e0, 0x5c13340, 0xc421a680e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc42248e1e0, 0x5c13340, 0xc421a680e0, 0xc420396440, 0xc420396460, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc42248e1e0, 0x5c13340, 0xc421a680e0, 0xc420396440, 0xc420396460, 0x6, 0xc424691968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc42248e1e0, 0xc421a68070, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0x5b8e380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422f61900, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422f61900, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422f61900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114367 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4219ab200, 0x15179d56f757b3a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421207c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4219ab200, 0x15179d56f757b3a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421207c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4219ab200, 0x15179d56f757b3a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421207c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4219ab1d0, 0x15179d56f757b3a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421207c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4219ab170, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421207c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4219ab170, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4219ab170, 0x5c4d11c, 0x10, 0xc4247bf9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4219ab170, 0xc423d7e6f8, 0xc4219ab170, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4219ab170, 0xc423d7e6f8, 0x1f, 0xc420a81630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422992930, 0xc423d7e6e0, 0x6a551e0, 0xc423ef86c0, 0xc400000008, 0xc423763a00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422992930, 0x6a551e0, 0xc423ef86c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422992930, 0x6a551e0, 0xc423ef86c0, 0xc423d7e630, 0xc423c49a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114963 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423b44d80, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b44d80, 0x15179d571a8ed6a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422058100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b44d80, 0x15179d571a8ed6a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422058100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423b44d80, 0x15179d571a8ed6a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422058100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423b44d50, 0x15179d571a8ed6a0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422058100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423b44cf0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422058100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423b44cf0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423b44c30, 0x5c4d11c, 0x10, 0xc4238e72c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423b44c30, 0xc4223cf2d0, 0xc423b44c30, 0xc422f4b568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423b44c30, 0xc4223cf2d0, 0xc420b53500, 0xc4241ca718, 0xc423aa12c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423b44a80, 0x5c13340, 0xc4223cf2d0, 0x5c13340, 0xc4223cf2d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423b44a80, 0x5c13340, 0xc4223cf2d0, 0xc42230a220, 0xc42230a240, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423b44a80, 0x5c13340, 0xc4223cf2d0, 0xc42230a220, 0xc42230a240, 0x6, 0xc422f4b968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423b44a80, 0xc4223cf260, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422b8dc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423510b40, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423510b40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423510b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114402 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422f61680, 0x45563f3, 0xc422d091e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422f61680, 0x5d2c558, 0xc422d09420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42248ed20, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42248ed20, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423d7ee88, 0x5b6ec40, 0xc4219e4900, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423ef8980, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423d7ee88, 0x5b6ec40, 0xc4219e4900, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc423ef8980, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423d7ee88, 0x5b6ec40, 0xc4219e4900, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423ef8980, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423d7ee88, 0x5b6ec40, 0xc4219e4900, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac9170, 0x6a551e0, 0xc423ef8980, 0xc423d7ee88, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x4015826) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421a698f0, 0xc423d7ee70, 0x6a551e0, 0xc423ef8980, 0xc421453998, 0x5d2d188, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421a698f0, 0x6a551e0, 0xc423ef8980, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421a698f0, 0x6a551e0, 0xc423ef8980, 0xc423d7edc0, 0xc4219e48a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114930 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379701) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4235dbaa0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4235dbaa0, 0x15179d571aa87150, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4235dbaa0, 0x15179d571aa87150, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4235dbaa0, 0x15179d571aa87150, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4237b6570, 0x15179d571aa87150, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4237b6510, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4237b6510, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4237b6510, 0x5c4d11c, 0x10, 0xc424dbf9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4237b6510, 0xc424a1fda8, 0xc4237b6510, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4237b6510, 0xc424a1fda8, 0x1f, 0xc42002fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420b616c0, 0xc424a1fd90, 0x6a551e0, 0xc42378ce00, 0xc400000008, 0xc421452cc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420b616c0, 0x6a551e0, 0xc42378ce00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420b616c0, 0x6a551e0, 0xc42378ce00, 0xc424a1fb80, 0xc420d7ac00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114929 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4228bae10, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4228bae10, 0x15179d571b487df8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42310c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4228bae10, 0x15179d571b487df8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42310c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4228bae10, 0x15179d571b487df8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42310c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4228bade0, 0x15179d571b487df8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42310c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4228bad80, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42310c900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4228bad80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4228bad80, 0x5c4d11c, 0x10, 0xc42524b9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4228bad80, 0xc422b640c8, 0xc4228bad80, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4228bad80, 0xc422b640c8, 0x1f, 0xc420892630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423d9d960, 0xc422b640b0, 0x6a551e0, 0xc422ceed80, 0xc400000008, 0xc4241cc600, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423d9d960, 0x6a551e0, 0xc422ceed80, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423d9d960, 0x6a551e0, 0xc422ceed80, 0xc423e71600, 0xc4228f2540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114546 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42130d560, 0xc4206a4230, 0x7, 0xc4224f0be0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42130d560, 0x7, 0xc4224f0be0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42130d560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42130d560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42130d560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42130d560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4247c4380, 0x6a552a0, 0xc42130d530, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc422f11d00, 0xc4250f3500, 0x0, 0xc422f11e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4250f3500, 0x1, 0xc42130d4a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421c35440, 0xc423945500, 0x5, 0x5, 0x15179d571375b708, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4213a50e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114947 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423b9b8c0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b9b8c0, 0x15179d5701e88150, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a6500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b9b8c0, 0x15179d5701e88150, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a6500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423b9b8c0, 0x15179d5701e88150, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a6500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423b9b890, 0x15179d5701e88150, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a6500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423b9b770, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a6500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423b9b770, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423b9b740, 0x5c4d11c, 0x10, 0xc4233b92c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423b9b740, 0xc4226160e0, 0xc423b9b740, 0xc425249568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423b9b740, 0xc4226160e0, 0xc420b53500, 0xc4204f8f80, 0xc424038740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423b9b620, 0x5c13340, 0xc4226160e0, 0x5c13340, 0xc4226160e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423b9b620, 0x5c13340, 0xc4226160e0, 0xc420349da0, 0xc420349dc0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423b9b620, 0x5c13340, 0xc4226160e0, 0xc420349da0, 0xc420349dc0, 0x6, 0xc425249968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423b9b620, 0xc422616070, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424879c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc421b1adc0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc421b1adc0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc421b1adc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114630 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc423773530, 0xc420be5f80, 0xc42252dc20, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423773530, 0x15179d57159199f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423773530, 0x15179d57159199f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423773530, 0x15179d57159199f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423773500, 0x15179d57159199f8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4237734a0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4237734a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4237734a0, 0x5c4d11c, 0x10, 0xc423e839b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4237734a0, 0xc423b4a858, 0xc4237734a0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4237734a0, 0xc423b4a858, 0x1f, 0xc420a83e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420be5f10, 0xc423b4a840, 0x6a551e0, 0xc423aa1680, 0xc400000008, 0xc42391c780, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420be5f10, 0x6a551e0, 0xc423aa1680, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420be5f10, 0x6a551e0, 0xc423aa1680, 0xc423b4a580, 0xc42252db60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115261 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc420d69a70, 0xc421081a40, 0xc423942240, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d69a70, 0x15179d5701d0e658, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d69a70, 0x15179d5701d0e658, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420d69a70, 0x15179d5701d0e658, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420d69a40, 0x15179d5701d0e658, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420d698c0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420d698c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420d69890, 0x5c4d11c, 0x10, 0xc42366d2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420d69890, 0xc4210819d0, 0xc420d69890, 0xc4230e1568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420d69890, 0xc4210819d0, 0xc420b53500, 0xc421ac8460, 0xc424a5ad80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc424597bf0, 0x5c13340, 0xc4210819d0, 0x5c13340, 0xc4210819d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc424597bf0, 0x5c13340, 0xc4210819d0, 0xc422d4da60, 0xc422d4da80, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc424597bf0, 0x5c13340, 0xc4210819d0, 0xc422d4da60, 0xc422d4da80, 0x6, 0xc4230e1968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc424597bf0, 0xc4210818f0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc420ccfc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422751400, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422751400, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422751400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114642 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4224c8780, 0x45563f3, 0xc421df31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4224c8780, 0x5d2c558, 0xc421df3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42377aa20, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42377aa20, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4237dac78, 0x5b6ec40, 0xc424201080, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424a16140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4237dac78, 0x5b6ec40, 0xc424201080, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424a16140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4237dac78, 0x5b6ec40, 0xc424201080, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424a16140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4237dac78, 0x5b6ec40, 0xc424201080, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac8bd8, 0x6a551e0, 0xc424a16140, 0xc4237dac78, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7f7a6925f) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423f2ea80, 0xc4237dac60, 0x6a551e0, 0xc424a16140, 0xc400000008, 0xc4232a40c0, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423f2ea80, 0x6a551e0, 0xc424a16140, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423f2ea80, 0x6a551e0, 0xc424a16140, 0xc423ba9340, 0xc424201020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 21509 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc42334d860, 0xc420518e60, 0x7, 0xc420bd3360, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc42334d860, 0x7, 0xc420bd3360, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc42334d860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e65900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc42334d860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e65900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc42334d860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e65900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc42334d860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e65800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423d31dc0, 0x6a552a0, 0xc42334d7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e65800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420e89200, 0x6a551e0, 0xc4241d2480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3ff0000000000000, 0xc420e65800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc423220820, 0x6a551e0, 0xc4241d2480, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:959 +0x3a0 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).sendEndTxnReq(0xc423220820, 0x6a551e0, 0xc4241d2480, 0x5d2b501, 0x0, 0x41a6460) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:671 +0x1be github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).commit(0xc423220820, 0x6a551e0, 0xc4241d2480, 0xc423220820, 0xc400000000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:509 +0x51 github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Commit(0xc423220820, 0x6a551e0, 0xc4241d2480, 0xc424ecfbb0, 0x4669c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:541 +0x3f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).handleAutoCommit(0xc422ee9000, 0x6a551e0, 0xc4241d2480, 0x6a568e0, 0xc4223242a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:927 +0x95 github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc422ee9000, 0x6a551e0, 0xc4241d2480, 0x6a568e0, 0xc4223242a0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:100 +0x51f github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc422ee9000, 0x6a551e0, 0xc422b57640, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:907 +0x1db1 github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42100a000, 0x6a551e0, 0xc422b57640, 0x0, 0x0, 0xc42316500d, 0x4, 0x0, 0x0, 0x6a434e0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:488 +0x1095 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl.func3(0xc4222d7cc0, 0xc42100a000, 0x6a551e0, 0xc422b57640, 0xc423644540, 0x5400, 0x15000, 0xc42136a190, 0xc42238b680, 0xc4222d7cb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:260 +0xfc created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:259 +0xf5b goroutine 114257 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420faf920, 0xc4206a4230, 0x7, 0xc4241eea50, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420faf920, 0x7, 0xc4241eea50, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420faf920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420faf920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420faf920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420faf920, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42435e000, 0x6a552a0, 0xc420faf890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424a0cf78, 0xc42235ae00, 0x0, 0xc4203c7320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42235ae00, 0x1, 0xc4203c7320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4243cc000, 0x44, 0x44, 0x15179d570f1cbd28, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4243cc000, 0x44, 0x44, 0x15179d570f1cbd28, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc422027f50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114732 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42234a510, 0x15179d5716bd2590, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f73800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42234a510, 0x15179d5716bd2590, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f73800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42234a510, 0x15179d5716bd2590, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f73800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42234a4b0, 0x15179d5716bd2590, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f73800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42234a390, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420f73800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42234a390, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42234a240, 0x5c4d11c, 0x10, 0xc4227cd2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42234a240, 0xc420b33030, 0xc42234a240, 0xc421983568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42234a240, 0xc420b33030, 0xc420b53500, 0xc421ac90d8, 0xc4214531c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc42234a0f0, 0x5c13340, 0xc420b33030, 0x5c13340, 0xc420b33030, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc42234a0f0, 0x5c13340, 0xc420b33030, 0xc42048f480, 0xc42048f4e0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc42234a0f0, 0x5c13340, 0xc420b33030, 0xc42048f480, 0xc42048f4e0, 0x6, 0xc421983968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc42234a0f0, 0xc420b32fc0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc4210a5c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42473c280, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42473c280, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc42473c280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115032 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421684090, 0xc4206a4230, 0x7, 0xc42341a690, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421684090, 0x7, 0xc42341a690, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421684090, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421684090, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421684090, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421684090, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42209a540, 0x6a552a0, 0xc421684060, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423e5fd00, 0xc424dc0e00, 0x0, 0xc423e5fe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424dc0e00, 0x1, 0xc421684000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42301eb40, 0xc420b53b00, 0x5, 0x5, 0x15179d571e8a33a8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4224e0ff0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115264 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4221e0000, 0x45563f3, 0xc423c871e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4221e0000, 0x5d2c558, 0xc423c87420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420bd45d0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420bd45d0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a0c8, 0x5b6ec40, 0xc42390a6c0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4231b7a00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a0c8, 0x5b6ec40, 0xc42390a6c0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc4231b7a00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a0c8, 0x5b6ec40, 0xc42390a6c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4231b7a00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a0c8, 0x5b6ec40, 0xc42390a6c0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f808e8, 0x6a551e0, 0xc4231b7a00, 0xc423a9a0c8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x80be69c72) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4200dfce0, 0xc423a9a0b0, 0x6a551e0, 0xc4231b7a00, 0xc400000008, 0xc4231b7ac0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4200dfce0, 0x6a551e0, 0xc4231b7a00, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4200dfce0, 0x6a551e0, 0xc4231b7a00, 0xc423a9a000, 0xc42390a5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115199 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423fbf710, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbf710, 0x15179d57226df9a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbf710, 0x15179d57226df9a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423fbf710, 0x15179d57226df9a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423fbf680, 0x15179d57226df9a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423fbf620, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423fbf620, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423fbf620, 0x5c4d11c, 0x10, 0xc425af39b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423fbf620, 0xc42474e018, 0xc423fbf620, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423fbf620, 0xc42474e018, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422667dc0, 0xc42474e000, 0x6a551e0, 0xc42335a280, 0xc400000008, 0xc424918b40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422667dc0, 0x6a551e0, 0xc42335a280, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422667dc0, 0x6a551e0, 0xc42335a280, 0xc42371fef0, 0xc42168d9e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115191 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4204e1260, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4204e1260, 0x15179d570d92c998, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42374c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4204e1260, 0x15179d570d92c998, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42374c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4204e1260, 0x15179d570d92c998, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42374c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4204e1170, 0x15179d570d92c998, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42374c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4204e0930, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42374c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4204e0930, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4204e08d0, 0x5c4d11c, 0x10, 0xc422f392c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4204e08d0, 0xc423f2f810, 0xc4204e08d0, 0xc42385b568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4204e08d0, 0xc423f2f810, 0xc420b53500, 0xc4204f82e0, 0xc42403a9c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4204e05a0, 0x5c13340, 0xc423f2f810, 0x5c13340, 0xc423f2f810, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4204e05a0, 0x5c13340, 0xc423f2f810, 0xc4215e3680, 0xc4215e36a0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4204e05a0, 0x5c13340, 0xc423f2f810, 0xc4215e3680, 0xc4215e36a0, 0x6, 0xc42385b968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4204e05a0, 0xc423f2f7a0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba6c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420915040, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420915040, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc420915040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115145 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423959040, 0x45563f3, 0xc4238df1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423959040, 0x5d2c558, 0xc4238df420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4245620c0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4245620c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc424709618, 0x5b6ec40, 0xc4226bbaa0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4241cc7c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc424709618, 0x5b6ec40, 0xc4226bbaa0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc4241cc7c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc424709618, 0x5b6ec40, 0xc4226bbaa0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4241cc7c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc424709618, 0x5b6ec40, 0xc4226bbaa0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca100, 0x6a551e0, 0xc4241cc7c0, 0xc424709618, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x80733e58f) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422758fc0, 0xc424709600, 0x6a551e0, 0xc4241cc7c0, 0xc400000008, 0xc424264f40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422758fc0, 0x6a551e0, 0xc4241cc7c0, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422758fc0, 0x6a551e0, 0xc4241cc7c0, 0xc4247093f0, 0xc4226bba40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115002 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42124c180, 0xc421045a40, 0xc423e0a960, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124c180, 0x15179d571d851fb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124c180, 0x15179d571d851fb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42124c180, 0x15179d571d851fb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42124c150, 0x15179d571d851fb8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42124c0f0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42124c0f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42124c0f0, 0x5c4d11c, 0x10, 0xc4230959b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42124c0f0, 0xc422a78858, 0xc42124c0f0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42124c0f0, 0xc422a78858, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4210459d0, 0xc422a78840, 0x6a551e0, 0xc4226c5880, 0xc400000008, 0xc4226c5740, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4210459d0, 0x6a551e0, 0xc4226c5880, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4210459d0, 0x6a551e0, 0xc4226c5880, 0xc422a786e0, 0xc423e0a900) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 111576 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42244de60, 0xc4206a4230, 0x7, 0xc4237bd7c0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42244de60, 0x7, 0xc4237bd7c0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42244de60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42244de60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42244de60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42244de60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc420cd1c00, 0x6a552a0, 0xc42244de00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc425326f78, 0xc422c5f800, 0x2, 0xc42439ce60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc422c5f800, 0x3, 0xc42439ce40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42198bb00, 0x30, 0x30, 0x15179d56ce36a620, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42198bb00, 0x30, 0x30, 0x15179d56ce36a620, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42244dda0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114317 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4213a40c0, 0x15179d570f278ac8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4213a40c0, 0x15179d570f278ac8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4213a40c0, 0x15179d570f278ac8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4213a4060, 0x15179d570f278ac8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4213a4000, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4213a4000, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4213a4000, 0x5c4d11c, 0x10, 0xc421c519b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4213a4000, 0xc424a434b8, 0xc4213a4000, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4213a4000, 0xc424a434b8, 0x1f, 0xc42002f630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420343490, 0xc424a434a0, 0x6a551e0, 0xc422ef4500, 0xc400000008, 0xc422281a40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420343490, 0x6a551e0, 0xc422ef4500, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420343490, 0x6a551e0, 0xc422ef4500, 0xc424a433f0, 0xc423d63560) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115061 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4233a8c80, 0x45563f3, 0xc4236291e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4233a8c80, 0x5d2c558, 0xc423629420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420c0f050, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420c0f050, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421b52dd8, 0x5b6ec40, 0xc4234187e0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424a5b300, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421b52dd8, 0x5b6ec40, 0xc4234187e0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc424a5b300, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421b52dd8, 0x5b6ec40, 0xc4234187e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424a5b300, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421b52dd8, 0x5b6ec40, 0xc4234187e0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f9958, 0x6a551e0, 0xc424a5b300, 0xc421b52dd8, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x9378ff8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42311cfc0, 0xc421b52dc0, 0x6a551e0, 0xc424a5b300, 0xc400000008, 0xc421452480, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42311cfc0, 0x6a551e0, 0xc424a5b300, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42311cfc0, 0x6a551e0, 0xc424a5b300, 0xc421b52d10, 0xc423418780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115239 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc423008ff0, 0xc420e5c0e0, 0xc422226a80, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423008ff0, 0x15179d56f757b3a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423008ff0, 0x15179d56f757b3a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423008ff0, 0x15179d56f757b3a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423008fc0, 0x15179d56f757b3a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423008f60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4200af000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423008f60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423008f60, 0x5c4d11c, 0x10, 0xc423b059b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423008f60, 0xc42474f358, 0xc423008f60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423008f60, 0xc42474f358, 0x1f, 0xc42002ee30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5c070, 0xc42474f340, 0x6a551e0, 0xc42335b640, 0xc400000008, 0xc42106ea80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5c070, 0x6a551e0, 0xc42335b640, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5c070, 0x6a551e0, 0xc42335b640, 0xc42474f1e0, 0xc4222269c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115256 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423875b80, 0x45563f3, 0xc4236291e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423875b80, 0x5d2c558, 0xc423629420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc424596690, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc424596690, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b314b8, 0x5b6ec40, 0xc422cd6cc0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423a47d00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b314b8, 0x5b6ec40, 0xc422cd6cc0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc423a47d00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b314b8, 0x5b6ec40, 0xc422cd6cc0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423a47d00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b314b8, 0x5b6ec40, 0xc422cd6cc0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80840, 0x6a551e0, 0xc423a47d00, 0xc423b314b8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x80d215e43) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e75d50, 0xc423b314a0, 0x6a551e0, 0xc423a47d00, 0xc400000008, 0xc421453480, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e75d50, 0x6a551e0, 0xc423a47d00, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e75d50, 0x6a551e0, 0xc423a47d00, 0xc423b313f0, 0xc422cd6c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115266 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4237fa640, 0x45563f3, 0xc4241c31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4237fa640, 0x5d2c558, 0xc4241c3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42124d050, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42124d050, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4231d32a8, 0x5b6ec40, 0xc423e0b860, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc422494cc0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4231d32a8, 0x5b6ec40, 0xc423e0b860, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc422494cc0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4231d32a8, 0x5b6ec40, 0xc423e0b860, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc422494cc0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4231d32a8, 0x5b6ec40, 0xc423e0b860, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca5c0, 0x6a551e0, 0xc422494cc0, 0xc4231d32a8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7c28e7b0c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42311c2a0, 0xc4231d3290, 0x6a551e0, 0xc422494cc0, 0xc400000008, 0xc4237629c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42311c2a0, 0x6a551e0, 0xc422494cc0, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42311c2a0, 0x6a551e0, 0xc422494cc0, 0xc4231d3130, 0xc423e0b7a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114715 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423e760c0, 0xc4206a4230, 0x7, 0xc4215c2640, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423e760c0, 0x7, 0xc4215c2640, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423e760c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423e760c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423e760c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423e760c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4230728c0, 0x6a552a0, 0xc423e76090, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42591cf78, 0xc4245fd800, 0x0, 0xc4203b2ad0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4245fd800, 0x1, 0xc4203b2ad0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424ccea00, 0x3a, 0x3a, 0x15179d57176cfd30, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424ccea00, 0x3a, 0x3a, 0x15179d57176cfd30, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4224f5f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114399 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422707590, 0xc4206a4230, 0x7, 0xc420c6ce10, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422707590, 0x7, 0xc420c6ce10, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422707590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422707590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422707590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422707590, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4224d4380, 0x6a552a0, 0xc422707560, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42306bd00, 0xc424b8ee00, 0x0, 0xc42306be38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424b8ee00, 0x1, 0xc4227074d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42399d560, 0xc426460300, 0x5, 0x5, 0x15179d5710f040c0, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc421a3bc50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114867 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423406b70, 0xc4206a4230, 0x7, 0xc42432ea00, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423406b70, 0x7, 0xc42432ea00, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423406b70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423406b70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423406b70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423406b70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e06380, 0x6a552a0, 0xc423406b40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc422bd3d00, 0xc420d5a000, 0x0, 0xc422bd3e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc420d5a000, 0x1, 0xc423406ae0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42399d9e0, 0xc421baa000, 0x5, 0x5, 0x15179d5719d4c9b8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42124f5c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115085 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc42334df50, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42334df50, 0x15179d571e522030, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42334df50, 0x15179d571e522030, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42334df50, 0x15179d571e522030, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42334dec0, 0x15179d571e522030, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42334de60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42334de60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42334de60, 0x5c4d11c, 0x10, 0xc4234639b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42334de60, 0xc42371f408, 0xc42334de60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42334de60, 0xc42371f408, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421037570, 0xc42371f3f0, 0x6a551e0, 0xc422495e00, 0xc400000008, 0xc42397d980, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421037570, 0x6a551e0, 0xc422495e00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421037570, 0x6a551e0, 0xc422495e00, 0xc42371f340, 0xc420e1af00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 110356 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422dacf60, 0xc4206a4230, 0x7, 0xc421641860, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422dacf60, 0x7, 0xc421641860, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422dacf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422dacf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422dacf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422dacf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4226908c0, 0x6a552a0, 0xc422dacf30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424480f78, 0xc4219a6000, 0x4, 0xc422d9fac0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4219a6000, 0x5, 0xc422d9fa80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc425454000, 0x3e, 0x3e, 0x15179d56b46bca18, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc425454000, 0x3e, 0x3e, 0x15179d56b46bca18, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4213a4ed0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114748 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423582a00, 0x45563f3, 0xc423ea51e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423582a00, 0x5d2c558, 0xc423ea5420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4225f8480, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4225f8480, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42276b828, 0x5b6ec40, 0xc4218642a0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42456e140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42276b828, 0x5b6ec40, 0xc4218642a0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc42456e140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42276b828, 0x5b6ec40, 0xc4218642a0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42456e140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42276b828, 0x5b6ec40, 0xc4218642a0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f94e8, 0x6a551e0, 0xc42456e140, 0xc42276b828, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x801dbb5a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420dd24d0, 0xc42276b810, 0x6a551e0, 0xc42456e140, 0xc400000008, 0xc42397c2c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420dd24d0, 0x6a551e0, 0xc42456e140, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420dd24d0, 0x6a551e0, 0xc42456e140, 0xc42276b550, 0xc4218641e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114478 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420bd4030, 0x15179d5712f92c38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420bd4030, 0x15179d5712f92c38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420bd4030, 0x15179d5712f92c38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420bd4000, 0x15179d5712f92c38, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420d69f80, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420d69f80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420d69f80, 0x5c4d11c, 0x10, 0xc422cf79b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420d69f80, 0xc4231d2908, 0xc420d69f80, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420d69f80, 0xc4231d2908, 0x1f, 0xc420bcb630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42311c690, 0xc4231d28f0, 0x6a551e0, 0xc4231b6d80, 0xc400000008, 0xc42397d500, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42311c690, 0x6a551e0, 0xc4231b6d80, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42311c690, 0x6a551e0, 0xc4231b6d80, 0xc4231d2840, 0xc420c71b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114384 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420bd5680, 0xc4206a4230, 0x7, 0xc4228f1270, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420bd5680, 0x7, 0xc4228f1270, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420bd5680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420bd5680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420bd5680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420bd5680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2ce00, 0x6a552a0, 0xc420bd5650, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421c76f78, 0xc421b46e00, 0x0, 0xc42164e170) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421b46e00, 0x1, 0xc42164e170) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x40, 0x40, 0x15179d571324d018, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x40, 0x40, 0x15179d571324d018, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4213a5020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114951 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421a95380, 0xc4206a4230, 0x7, 0xc42350b3b0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421a95380, 0x7, 0xc42350b3b0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421a95380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421a95380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421a95380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421a95380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4249cee00, 0x6a552a0, 0xc421a95350, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4236c3d00, 0xc424a73100, 0x0, 0xc4236c3e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424a73100, 0x1, 0xc421a952f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42245d7a0, 0xc421161800, 0x5, 0x5, 0x15179d571b95a010, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4228d0cf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114575 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4220972c0, 0x45563f3, 0xc42313d1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4220972c0, 0x5d2c558, 0xc42313d420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc423b44a50, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc423b44a50, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423c1bae8, 0x5b6ec40, 0xc4228f32c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4238be740, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423c1bae8, 0x5b6ec40, 0xc4228f32c0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc4238be740, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423c1bae8, 0x5b6ec40, 0xc4228f32c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4238be740, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423c1bae8, 0x5b6ec40, 0xc4228f32c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80a28, 0x6a551e0, 0xc4238be740, 0xc423c1bae8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7f09ad284) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212473b0, 0xc423c1bad0, 0x6a551e0, 0xc4238be740, 0xc400000008, 0xc423315e40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212473b0, 0x6a551e0, 0xc4238be740, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212473b0, 0x6a551e0, 0xc4238be740, 0xc423c1b550, 0xc4228f31a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115171 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423b9a7e0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b9a7e0, 0x15179d57209d6bd8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b9a7e0, 0x15179d57209d6bd8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423b9a7e0, 0x15179d57209d6bd8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423b9a7b0, 0x15179d57209d6bd8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423b9a750, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423b9a750, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423b9a750, 0x5c4d11c, 0x10, 0xc42309d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423b9a750, 0xc422c7f828, 0xc423b9a750, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423b9a750, 0xc422c7f828, 0x1f, 0xc420895630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4223071f0, 0xc422c7f810, 0x6a551e0, 0xc4214535c0, 0xc400000008, 0xc424a16380, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4223071f0, 0x6a551e0, 0xc4214535c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4223071f0, 0x6a551e0, 0xc4214535c0, 0xc422c7f550, 0xc4235b9380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114751 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b14420, 0x15179d56feda37b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4234e2400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b14420, 0x15179d56feda37b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4234e2400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422b14420, 0x15179d56feda37b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4234e2400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422b143f0, 0x15179d56feda37b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4234e2400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422b14390, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4234e2400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422b14390, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422b142d0, 0x5c4d11c, 0x10, 0xc42312b2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422b142d0, 0xc420dd3500, 0xc422b142d0, 0xc42591d568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422b142d0, 0xc420dd3500, 0xc420b53500, 0xc4204f9550, 0xc42456eb00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc422b14090, 0x5c13340, 0xc420dd3500, 0x5c13340, 0xc420dd3500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc422b14090, 0x5c13340, 0xc420dd3500, 0xc42037c580, 0xc42037c5e0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc422b14090, 0x5c13340, 0xc420dd3500, 0xc42037c580, 0xc42037c5e0, 0x6, 0xc42591d968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc422b14090, 0xc420dd33b0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba6c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4235837c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4235837c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4235837c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 112345 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4216843c0, 0xc4206a4230, 0x7, 0xc42341a8c0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4216843c0, 0x7, 0xc42341a8c0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4216843c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4216843c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4216843c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4216843c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42209b180, 0x6a552a0, 0xc421684390, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a55260, 0xc422b687e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a55260, 0xc422b687e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a55260, 0xc422b687e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a55260, 0xc422b687e0, 0xc4245c9a80, 0xc424dc1500, 0x6cae460, 0x6a55260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a55260, 0xc422b687e0, 0xc424dc1500, 0xc422b687e0, 0xc4211a33f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423969d40, 0x4, 0x4, 0x15179d56dc600101, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:625 +0x7a2 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42517e500, 0x3e, 0x3e, 0x15179d56dc60e670, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:365 +0x196 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4212e2840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114769 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4223f2330, 0xc4206a4230, 0x7, 0xc4215c35e0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4223f2330, 0x7, 0xc4215c35e0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4223f2330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4223f2330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4223f2330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4223f2330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4248388c0, 0x6a552a0, 0xc4223f2300, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424911d00, 0xc424188000, 0x0, 0xc424911e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424188000, 0x1, 0xc4223f22a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4222a4360, 0xc4205c6f00, 0x5, 0x5, 0x15179d571971be40, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc421180300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115138 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423b4de00, 0x45563f3, 0xc424b6f1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423b4de00, 0x5d2c558, 0xc424b6f420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4223d56e0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4223d56e0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9ba38, 0x5b6ec40, 0xc423d62f00, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424e861c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9ba38, 0x5b6ec40, 0xc423d62f00, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc424e861c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9ba38, 0x5b6ec40, 0xc423d62f00, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424e861c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9ba38, 0x5b6ec40, 0xc423d62f00, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f809e8, 0x6a551e0, 0xc424e861c0, 0xc423a9ba38, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x807c9da0f) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420ceb810, 0xc423a9ba20, 0x6a551e0, 0xc424e861c0, 0xc400000008, 0xc421452980, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420ceb810, 0x6a551e0, 0xc424e861c0, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420ceb810, 0x6a551e0, 0xc424e861c0, 0xc423a9b970, 0xc423d62e40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115267 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42359db30, 0xc421080f50, 0xc42395ce40, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359db30, 0x15179d56f99c9a90, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359db30, 0x15179d56f99c9a90, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42359db30, 0x15179d56f99c9a90, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42359dad0, 0x15179d56f99c9a90, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42359da70, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc420721600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42359da70, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42359da40, 0x5c4d11c, 0x10, 0xc4231932c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42359da40, 0xc421080ee0, 0xc42359da40, 0xc4230e1568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42359da40, 0xc421080ee0, 0xc420b53500, 0xc421ac83f0, 0xc424a5a240) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc42124d440, 0x5c13340, 0xc421080ee0, 0x5c13340, 0xc421080ee0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc42124d440, 0x5c13340, 0xc421080ee0, 0xc422d4d120, 0xc422d4d140, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc42124d440, 0x5c13340, 0xc421080ee0, 0xc422d4d120, 0xc422d4d140, 0x6, 0xc4230e1968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc42124d440, 0xc421080e70, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc421131c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4237fa8c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4237fa8c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4237fa8c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 110761 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423123890, 0xc4206a4230, 0x7, 0xc4224ce280, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423123890, 0x7, 0xc4224ce280, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423123890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423123890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423123890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423123890, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423645a40, 0x6a552a0, 0xc423123860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc422c02f78, 0xc4241ff800, 0x7, 0xc4228cd6f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4241ff800, 0x5, 0xc4228cd680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4266e8000, 0x44, 0x44, 0x15179d56bb8aa288, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4266e8000, 0x44, 0x44, 0x15179d56bb8aa288, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4222ccdb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 113937 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4221060f0, 0xc4206a4230, 0x7, 0xc4233d4190, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4221060f0, 0x7, 0xc4233d4190, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4221060f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4221060f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4221060f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4221060f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc420cd1880, 0x6a552a0, 0xc4221060c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424ce0f78, 0xc42163c000, 0x6, 0xc4231fa060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42163c000, 0x5, 0xc4231fa000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x3c, 0x3c, 0x15179d57052fc0f8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x3c, 0x3c, 0x15179d57052fc0f8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc422106060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115300 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4221e0a00, 0x45563f3, 0xc4243591e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4221e0a00, 0x5d2c558, 0xc424359420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420bd59b0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420bd59b0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a7a8, 0x5b6ec40, 0xc42390b0e0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc422e306c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a7a8, 0x5b6ec40, 0xc42390b0e0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc422e306c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a7a8, 0x5b6ec40, 0xc42390b0e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc422e306c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a7a8, 0x5b6ec40, 0xc42390b0e0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80938, 0x6a551e0, 0xc422e306c0, 0xc423a9a7a8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x800c49042) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422528850, 0xc423a9a790, 0x6a551e0, 0xc422e306c0, 0xc400000008, 0xc422e30740, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422528850, 0x6a551e0, 0xc422e306c0, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422528850, 0x6a551e0, 0xc422e306c0, 0xc423a9a630, 0xc42390b020) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114938 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4238c6120, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4238c6120, 0x15179d571ad44410, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4238c6120, 0x15179d571ad44410, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4238c6120, 0x15179d571ad44410, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4238c60f0, 0x15179d571ad44410, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4238c6090, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4238c6090, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4238c6090, 0x5c4d11c, 0x10, 0xc4219919b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4238c6090, 0xc422c7f1f8, 0xc4238c6090, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4238c6090, 0xc422c7f1f8, 0x1f, 0xc42002fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42454e070, 0xc422c7f1e0, 0x6a551e0, 0xc424a16dc0, 0xc400000008, 0xc422e312c0, 0x299a73a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42454e070, 0x6a551e0, 0xc424a16dc0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42454e070, 0x6a551e0, 0xc424a16dc0, 0xc422c7e4d0, 0xc4211c0d20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114521 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4233a8780, 0x45563f3, 0xc424e2b1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4233a8780, 0x5d2c558, 0xc424e2b420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42130f740, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42130f740, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474e908, 0x5b6ec40, 0xc42363b5c0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424a5b240, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474e908, 0x5b6ec40, 0xc42363b5c0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc424a5b240, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474e908, 0x5b6ec40, 0xc42363b5c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424a5b240, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42474e908, 0x5b6ec40, 0xc42363b5c0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f805f8, 0x6a551e0, 0xc424a5b240, 0xc42474e908, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7fc73d608) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421037dc0, 0xc42474e8f0, 0x6a551e0, 0xc424a5b240, 0xc400000008, 0xc4232a46c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421037dc0, 0x6a551e0, 0xc424a5b240, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421037dc0, 0x6a551e0, 0xc424a5b240, 0xc42474e580, 0xc42363b560) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 21348 [IO wait]: internal/poll.runtime_pollWait(0x77240c0, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc423641a18, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc423641a18, 0xc4216c8800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc423641a00, 0xc4216c8800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc423641a00, 0xc4216c8800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4206eedc0, 0xc4216c8800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc4236195e0, 0xc4216c8800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc422b15c20, 0x73ccbf8, 0xc4236195e0, 0x5, 0xc4236195e0, 0xbc) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc4224eca80, 0x5d2d117, 0xc4224ecba0, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc4224eca80, 0xc421c32000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc422aa2000, 0xc421c32000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4236445a0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4236445a0, 0xbe9de8a304c7b588, 0x7a5214e1a, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4236446d0, 0x6a46020, 0xc4236445a0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc423644540, 0x6a551e0, 0xc422b57600, 0xc4222d7c90, 0xc42100a000, 0x5400, 0x15000, 0xc42136a190, 0xc420f747e0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc421f2e8c0, 0x6a611e0, 0xc4224eca80, 0x0, 0x0, 0xc42316500d, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc42136a000, 0x6a551e0, 0xc421f2e8c0, 0x6a611e0, 0xc4224eca80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc421be2ff0, 0x6a61a80, 0xc4236195e0, 0xc422378470, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc4236195e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc420f747e0, 0x6a552a0, 0xc420f6d500, 0xc4205b0080, 0x6a61a80, 0xc4236195e0, 0xc420f6d530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 115259 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc424597470, 0xc4200deaf0, 0xc422cd7920, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc424597470, 0x15179d5701de3498, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc424597470, 0x15179d5701de3498, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc424597470, 0x15179d5701de3498, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc424597440, 0x15179d5701de3498, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4245973e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421e2b800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4245973e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4245973b0, 0x5c4d11c, 0x10, 0xc424f652c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4245973b0, 0xc4200de7e0, 0xc4245973b0, 0xc4267ed568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4245973b0, 0xc4200de7e0, 0xc420b53500, 0xc423f80878, 0xc4231b6780) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4245972c0, 0x5c13340, 0xc4200de7e0, 0x5c13340, 0xc4200de7e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4245972c0, 0x5c13340, 0xc4200de7e0, 0xc421b1d940, 0xc421b1d960, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4245972c0, 0x5c13340, 0xc4200de7e0, 0xc421b1d940, 0xc421b1d960, 0x6, 0xc4267ed968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4245972c0, 0xc4200de5b0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424926c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422750dc0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422750dc0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422750dc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114539 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc421863b90, 0xc4204a2620, 0xc4237b1560, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421863b90, 0x15179d5713fd2ad0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421863b90, 0x15179d5713fd2ad0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421863b90, 0x15179d5713fd2ad0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421863b60, 0x15179d5713fd2ad0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421863b00, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421863b00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421863b00, 0x5c4d11c, 0x10, 0xc422f2d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421863b00, 0xc423e2b778, 0xc421863b00, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421863b00, 0xc423e2b778, 0x1f, 0xc420bcb630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4204a25b0, 0xc423e2b760, 0x6a551e0, 0xc4226c5d40, 0xc400000008, 0xc422494dc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4204a25b0, 0x6a551e0, 0xc4226c5d40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4204a25b0, 0x6a551e0, 0xc4226c5d40, 0xc423e2b6b0, 0xc4237b1500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115109 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc422752060, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422752060, 0x15179d571f0249d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422752060, 0x15179d571f0249d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422752060, 0x15179d571f0249d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422752000, 0x15179d571f0249d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420163e60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420163e60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420163e60, 0x5c4d11c, 0x10, 0xc423d579b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420163e60, 0xc421f81988, 0xc420163e60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420163e60, 0xc421f81988, 0x1f, 0xc420897e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422666310, 0xc421f81970, 0x6a551e0, 0xc42335b100, 0xc400000008, 0xc422a5b740, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422666310, 0x6a551e0, 0xc42335b100, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422666310, 0x6a551e0, 0xc42335b100, 0xc421f81340, 0xc421ed4480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 62756 [IO wait]: internal/poll.runtime_pollWait(0x7725380, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc422d9ec98, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc422d9ec98, 0xc421f48800, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc422d9ec80, 0xc421f48800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc422d9ec80, 0xc421f48800, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc423f81330, 0xc421f48800, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc42348e640, 0xc421f48800, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc42139b890, 0x73ccbf8, 0xc42348e640, 0x5, 0xc42348e640, 0xc8) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc422dd2700, 0x5d2d117, 0xc422dd2820, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc422dd2700, 0xc422f09000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc4228a07c0, 0xc422f09000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4249ceae0) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4249ceae0, 0xbe9de8a3140bc660, 0x7b46540a8, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4249cec10, 0x6a46020, 0xc4249ceae0, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4249cea80, 0x6a551e0, 0xc42439ca80, 0xc422b05140, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc42456e840, 0x6a611e0, 0xc422dd2700, 0x0, 0x0, 0xc423bcb04a, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc42456e840, 0x6a611e0, 0xc422dd2700, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc42139b830, 0x6a61a80, 0xc42348e640, 0xc424bd56a0, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc42348e640) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc42348e640, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 114962 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4235108c0, 0x45563f3, 0xc4241151e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4235108c0, 0x5d2c558, 0xc424115420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc423b44480, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc423b44480, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423c1add8, 0x5b6ec40, 0xc42193f740, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423aa1140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423c1add8, 0x5b6ec40, 0xc42193f740, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc423aa1140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423c1add8, 0x5b6ec40, 0xc42193f740, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423aa1140, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423c1add8, 0x5b6ec40, 0xc42193f740, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca710, 0x6a551e0, 0xc423aa1140, 0xc423c1add8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x8059d486c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4223cf110, 0xc423c1adc0, 0x6a551e0, 0xc423aa1140, 0xc400000008, 0xc423aa0800, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4223cf110, 0x6a551e0, 0xc423aa1140, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4223cf110, 0x6a551e0, 0xc423aa1140, 0xc423c1a6e0, 0xc42193f6e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115103 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423b7eae0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b7eae0, 0x15179d570cc20c18, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423ace100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b7eae0, 0x15179d570cc20c18, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423ace100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423b7eae0, 0x15179d570cc20c18, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423ace100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423b7ea80, 0x15179d570cc20c18, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423ace100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423b7e9c0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423ace100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423b7e9c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423b7e960, 0x5c4d11c, 0x10, 0xc42309d2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423b7e960, 0xc422306380, 0xc423b7e960, 0xc4230df568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423b7e960, 0xc422306380, 0xc420b53500, 0xc423f80028, 0xc421452180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423b7e720, 0x5c13340, 0xc422306380, 0x5c13340, 0xc422306380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423b7e720, 0x5c13340, 0xc422306380, 0xc4223d64e0, 0xc4223d6500, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423b7e720, 0x5c13340, 0xc422306380, 0xc4223d64e0, 0xc4223d6500, 0x6, 0xc4230df968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423b7e720, 0xc4223062a0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc421b07c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4241b2500, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4241b2500, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc4241b2500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 113178 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423b446f0, 0xc4206a4230, 0x7, 0xc4211e59a0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423b446f0, 0x7, 0xc4211e59a0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423b446f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423b446f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423b446f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423b446f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c26000, 0x6a552a0, 0xc423b446c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4228c8f78, 0xc4219a8300, 0x2, 0xc424dcf820) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4219a8300, 0x3, 0xc424dcf800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424734000, 0x34, 0x34, 0x15179d56f3f527b0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424734000, 0x34, 0x34, 0x15179d56f3f527b0, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4236851d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115136 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4225b46c0, 0xc4233d2770, 0xc42414f320, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4225b46c0, 0x15179d572072f4c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4225b46c0, 0x15179d572072f4c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4225b46c0, 0x15179d572072f4c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4225b45d0, 0x15179d572072f4c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4225b44e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4225b44e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4225b44e0, 0x5c4d11c, 0x10, 0xc4243879b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4225b44e0, 0xc4218b5ae8, 0xc4225b44e0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4225b44e0, 0xc4218b5ae8, 0x1f, 0xc420bc6e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4233d2700, 0xc4218b5ad0, 0x6a551e0, 0xc42378d040, 0xc400000008, 0xc42391c140, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4233d2700, 0x6a551e0, 0xc42378d040, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4233d2700, 0x6a551e0, 0xc42378d040, 0xc4218b5760, 0xc42414f2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115285 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc420d690e0, 0xc421081880, 0xc42395dd40, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d690e0, 0x15179d5700f19a48, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420d690e0, 0x15179d5700f19a48, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420d690e0, 0x15179d5700f19a48, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420d69080, 0x15179d5700f19a48, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420d68f90, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc421314b00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420d68f90, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420d68f60, 0x5c4d11c, 0x10, 0xc4236652c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420d68f60, 0xc421081810, 0xc420d68f60, 0xc4230e1568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420d68f60, 0xc421081810, 0xc420b53500, 0xc421ac8440, 0xc424a5ab40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420d68cf0, 0x5c13340, 0xc421081810, 0x5c13340, 0xc421081810, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420d68cf0, 0x5c13340, 0xc421081810, 0xc422d4d980, 0xc422d4d9a0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420d68cf0, 0x5c13340, 0xc421081810, 0xc422d4d980, 0xc422d4d9a0, 0x6, 0xc4230e1968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420d68cf0, 0xc421081730, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc420ccfc40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423643540, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423643540, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423643540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115265 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4224e0510, 0xc42311d110, 0xc4238ee660, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0510, 0x15179d57053ead48, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120ca00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0510, 0x15179d57053ead48, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120ca00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224e0510, 0x15179d57053ead48, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120ca00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224e04e0, 0x15179d57053ead48, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120ca00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224e0480, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120ca00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224e0480, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224e0450, 0x5c4d11c, 0x10, 0xc4233e32c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224e0450, 0xc42311d0a0, 0xc4224e0450, 0xc425463568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224e0450, 0xc42311d0a0, 0xc420b53500, 0xc4241ca628, 0xc422495b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420bd4de0, 0x5c13340, 0xc42311d0a0, 0x5c13340, 0xc42311d0a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420bd4de0, 0x5c13340, 0xc42311d0a0, 0xc422e27c60, 0xc422e27ca0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420bd4de0, 0x5c13340, 0xc42311d0a0, 0xc422e27c60, 0xc422e27ca0, 0x6, 0xc425463968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420bd4de0, 0xc42311d030, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc421b04c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4221e0280, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4221e0280, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4221e0280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115141 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423b7e210, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b7e210, 0x15179d571f0756d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b7e210, 0x15179d571f0756d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423b7e210, 0x15179d571f0756d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423b7e1b0, 0x15179d571f0756d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423b7e120, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423b7e120, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423b7e120, 0x5c4d11c, 0x10, 0xc423a1b9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423b7e120, 0xc423a9bf08, 0xc423b7e120, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423b7e120, 0xc423a9bf08, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420cebe30, 0xc423a9bef0, 0x6a551e0, 0xc424e86600, 0xc400000008, 0xc42160e500, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420cebe30, 0x6a551e0, 0xc424e86600, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420cebe30, 0x6a551e0, 0xc424e86600, 0xc423a9be40, 0xc423d63bc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114352 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420163ce0, 0x15179d56f6ff9508, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421654f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420163ce0, 0x15179d56f6ff9508, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421654f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420163ce0, 0x15179d56f6ff9508, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421654f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420163cb0, 0x15179d56f6ff9508, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421654f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420163c50, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421654f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420163c50, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420163c50, 0x5c4d11c, 0x10, 0xc4247bf9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420163c50, 0xc422c82438, 0xc420163c50, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420163c50, 0xc422c82438, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f710a0, 0xc422c82420, 0x6a551e0, 0xc42141c880, 0xc400000008, 0xc423314780, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f710a0, 0x6a551e0, 0xc42141c880, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f710a0, 0x6a551e0, 0xc42141c880, 0xc422c82370, 0xc4223092c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113391 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4213a5230, 0xc4206a4230, 0x7, 0xc4228a6910, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4213a5230, 0x7, 0xc4228a6910, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4213a5230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4213a5230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4213a5230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4213a5230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2da40, 0x6a552a0, 0xc4213a5200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423063d00, 0xc42484ae00, 0x0, 0xc423063e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42484ae00, 0x1, 0xc4213a5170) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421eb8360, 0xc420c86c00, 0x5, 0x5, 0x15179d56f6f509d0, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423b4faa0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114554 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422aa1540, 0x45563f3, 0xc4234b31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422aa1540, 0x5d2c558, 0xc4234b3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420f64b40, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420f64b40, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422b65ae8, 0x5b6ec40, 0xc42353b320, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc422e30e40, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422b65ae8, 0x5b6ec40, 0xc42353b320, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc422e30e40, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422b65ae8, 0x5b6ec40, 0xc42353b320, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc422e30e40, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422b65ae8, 0x5b6ec40, 0xc42353b320, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca408, 0x6a551e0, 0xc422e30e40, 0xc422b65ae8, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x937bca8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423746e70, 0xc422b65ad0, 0x6a551e0, 0xc422e30e40, 0xc400000008, 0xc424a74b40, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423746e70, 0x6a551e0, 0xc422e30e40, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423746e70, 0x6a551e0, 0xc422e30e40, 0xc422b65970, 0xc42353b2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114542 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423710d50, 0xc4206a4230, 0x7, 0xc4217c7040, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423710d50, 0x7, 0xc4217c7040, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423710d50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423710d50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423710d50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423710d50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c3b340, 0x6a552a0, 0xc423710d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424559d00, 0xc423de1c00, 0x0, 0xc424559e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423de1c00, 0x1, 0xc423710cc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421b647e0, 0xc4222bcf00, 0x5, 0x5, 0x15179d5714426190, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc421f52930) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114443 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42166ba10, 0x15179d5711b2b600, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42166ba10, 0x15179d5711b2b600, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42166ba10, 0x15179d5711b2b600, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42166b710, 0x15179d5711b2b600, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42166b6b0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42166b6b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42166b6b0, 0x5c4d11c, 0x10, 0xc425b139b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42166b6b0, 0xc423f5efe8, 0xc42166b6b0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42166b6b0, 0xc423f5efe8, 0x1f, 0xc420bcb630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422992f50, 0xc423f5efd0, 0x6a551e0, 0xc42335a880, 0xc400000008, 0xc423a47040, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422992f50, 0x6a551e0, 0xc42335a880, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422992f50, 0x6a551e0, 0xc42335a880, 0xc423f5ef20, 0xc4231406c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 34912 [IO wait]: internal/poll.runtime_pollWait(0x7723b80, 0x72, 0x0) /Users/andrei/work/src/go/src/runtime/netpoll.go:173 +0x57 internal/poll.(*pollDesc).wait(0xc420f43618, 0x72, 0xffffffffffffff00, 0x6a3eea0, 0x6a32520) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:85 +0xae internal/poll.(*pollDesc).waitRead(0xc420f43618, 0xc420f3c000, 0x400, 0x400) /Users/andrei/work/src/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d internal/poll.(*FD).Read(0xc420f43600, 0xc420f3c000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/internal/poll/fd_unix.go:125 +0x18a net.(*netFD).Read(0xc420f43600, 0xc420f3c000, 0x400, 0x400, 0x1, 0x6a3afe0, 0x16d06cf0740) /Users/andrei/work/src/go/src/net/fd_unix.go:202 +0x52 net.(*conn).Read(0xc4241caef8, 0xc420f3c000, 0x400, 0x400, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/net/net.go:176 +0x6d github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux.(*MuxConn).Read(0xc421f50640, 0xc420f3c000, 0x400, 0x400, 0x5b473a0, 0x6cf0700, 0x73ccbf8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/cmux/cmux.go:218 +0x121 crypto/tls.(*block).readFromUntil(0xc42244dad0, 0x73ccbf8, 0xc421f50640, 0x5, 0xc421f50640, 0x9d) /Users/andrei/work/src/go/src/crypto/tls/conn.go:488 +0x95 crypto/tls.(*Conn).readRecord(0xc421f3dc00, 0x5d2d117, 0xc421f3dd20, 0x42f5bd1) /Users/andrei/work/src/go/src/crypto/tls/conn.go:590 +0xe0 crypto/tls.(*Conn).Read(0xc421f3dc00, 0xc4232b4000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/go/src/crypto/tls/conn.go:1134 +0x110 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*readTimeoutConn).Read(0xc421095ca0, 0xc4232b4000, 0x1000, 0x1000, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/v3.go:94 +0x11d bufio.(*Reader).fill(0xc4237cf560) /Users/andrei/work/src/go/src/bufio/bufio.go:97 +0x11a bufio.(*Reader).ReadByte(0xc4237cf560, 0xbe9de8a32d8d31a0, 0x7cde67b87, 0x6cae460) /Users/andrei/work/src/go/src/bufio/bufio.go:239 +0x39 github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase.(*ReadBuffer).ReadTypedMsg(0xc4237cf690, 0x6a46020, 0xc4237cf560, 0x6a448a0, 0x6cf0d10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/pgwirebase/encoding.go:119 +0x35 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).serveImpl(0xc4237cf500, 0x6a551e0, 0xc422280380, 0xc4210478d0, 0xc4204a4700, 0x5400, 0x15000, 0xc420724a90, 0xc4204ba360, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:277 +0x5e7 github.com/cockroachdb/cockroach/pkg/sql/pgwire.serveConn(0x6a551e0, 0xc421453840, 0x6a611e0, 0xc421f3dc00, 0x0, 0x0, 0xc422172022, 0x4, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:155 +0x207 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2(0xc420724900, 0x6a551e0, 0xc421453840, 0x6a611e0, 0xc421f3dc00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/server.go:573 +0x742 github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*Server).ServeConn2-fm(0x6a552a0, 0xc42244daa0, 0x6a61a80, 0xc421f50640, 0xc4231bac60, 0xf) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1283 +0x52 github.com/cockroachdb/cockroach/pkg/server.(*Server).Start.func17.1(0x6a61a80, 0xc421f50640) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/server.go:1287 +0x194 github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith.func1(0xc4204ba360, 0x6a552a0, 0xc420cbeb10, 0xc42000e3a0, 0x6a61a80, 0xc421f50640, 0xc420cbeb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:140 +0xa9 created by github.com/cockroachdb/cockroach/pkg/util/netutil.(*Server).ServeWith /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/netutil/net.go:137 +0x263 goroutine 115258 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422750640, 0x45563f3, 0xc4241151e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422750640, 0x5d2c558, 0xc424115420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc424596ea0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc424596ea0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31778, 0x5b6ec40, 0xc422cd75c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4231b63c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31778, 0x5b6ec40, 0xc422cd75c0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc4231b63c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31778, 0x5b6ec40, 0xc422cd75c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4231b63c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423b31778, 0x5b6ec40, 0xc422cd75c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80870, 0x6a551e0, 0xc4231b63c0, 0xc423b31778, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x80d215e43) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4200de460, 0xc423b31760, 0x6a551e0, 0xc4231b63c0, 0xc400000008, 0xc423910700, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4200de460, 0x6a551e0, 0xc4231b63c0, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4200de460, 0x6a551e0, 0xc4231b63c0, 0xc423b31550, 0xc422cd7440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114329 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420faf590, 0x15179d570e84f998, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420faf590, 0x15179d570e84f998, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420faf590, 0x15179d570e84f998, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420faf560, 0x15179d570e84f998, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420faf500, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422e8d400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420faf500, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420faf4d0, 0x5c4d11c, 0x10, 0xc4239312c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420faf4d0, 0xc423939340, 0xc420faf4d0, 0xc424f3b568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420faf4d0, 0xc423939340, 0xc420b53500, 0xc4204f8cd8, 0xc4241d3700) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420faeff0, 0x5c13340, 0xc423939340, 0x5c13340, 0xc423939340, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420faeff0, 0x5c13340, 0xc423939340, 0xc421b1c8e0, 0xc421b1c900, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420faeff0, 0x5c13340, 0xc423939340, 0xc421b1c8e0, 0xc421b1c900, 0x6, 0xc424f3b968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420faeff0, 0xc4239392d0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ad0c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc424285540, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc424285540, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc424285540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115209 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc4219f6d80, 0xc4202d8b90, 0x7, 0xc423081310, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc4219f6d80, 0x7, 0xc423081310, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc4219f6d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc4219f6d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc4219f6d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc4219f6d80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc422e07180, 0x6a552a0, 0xc423fbea80, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc422e07180, 0x6a552a0, 0xc423fbea80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc420fae3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc420fae3c0, 0xc424208450, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc4203b3c70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 113452 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4223d4090, 0x15179d56f76f3728, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4223d4090, 0x15179d56f76f3728, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4223d4090, 0x15179d56f76f3728, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422707fb0, 0x15179d56f76f3728, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422707f20, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422707f20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422707f20, 0x5c4d11c, 0x10, 0xc424a379b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422707f20, 0xc421f80a68, 0xc422707f20, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422707f20, 0xc421f80a68, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420ceb6c0, 0xc421f80a50, 0x6a551e0, 0xc424908040, 0xc400000008, 0xc42141dc00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420ceb6c0, 0x6a551e0, 0xc424908040, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420ceb6c0, 0x6a551e0, 0xc424908040, 0xc421f80840, 0xc422b1a1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114950 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4228d0a50, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4228d0a50, 0x15179d57053ead48, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a7400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4228d0a50, 0x15179d57053ead48, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a7400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4228d0a50, 0x15179d57053ead48, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a7400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4228d0a20, 0x15179d57053ead48, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a7400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4228d0990, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4208a7400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4228d0990, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4228d0960, 0x5c4d11c, 0x10, 0xc4247492c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4228d0960, 0xc422617490, 0xc4228d0960, 0xc42316b568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4228d0960, 0xc422617490, 0xc420b53500, 0xc4204f9070, 0xc424039200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4228d0810, 0x5c13340, 0xc422617490, 0x5c13340, 0xc422617490, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4228d0810, 0x5c13340, 0xc422617490, 0xc422b18d00, 0xc422b18d20, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4228d0810, 0x5c13340, 0xc422617490, 0xc422b18d00, 0xc422b18d20, 0x6, 0xc42316b968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4228d0810, 0xc422617420, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0x5b8e380) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc421e16140, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc421e16140, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc421e16140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115137 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423b9a690, 0xc4206a4230, 0x7, 0xc42128f540, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423b9a690, 0x7, 0xc42128f540, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423b9a690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423b9a690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423b9a690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423b9a690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ee5180, 0x6a552a0, 0xc423b9a660, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424a27d00, 0xc421948700, 0x0, 0xc424a27e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421948700, 0x1, 0xc423b9a600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420c79440, 0xc420fcc300, 0x5, 0x5, 0x15179d572076b5b0, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4225b4870) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114078 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423685b30, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423685b30, 0x15179d570ce9f6d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423685b30, 0x15179d570ce9f6d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423685b30, 0x15179d570ce9f6d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423685a70, 0x15179d570ce9f6d8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4236859b0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4236859b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4236859b0, 0x5c4d11c, 0x10, 0xc4231799b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4236859b0, 0xc4237e5b98, 0xc4236859b0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4236859b0, 0xc4237e5b98, 0x1f, 0xc42002f630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423d9ccb0, 0xc4237e5b80, 0x6a551e0, 0xc4238beec0, 0xc400000008, 0xc42160f7c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423d9ccb0, 0x6a551e0, 0xc4238beec0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423d9ccb0, 0x6a551e0, 0xc4238beec0, 0xc4237e48f0, 0xc423409680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114307 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4224e1b90, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e1b90, 0x15179d570ebac320, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e1b90, 0x15179d570ebac320, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224e1b90, 0x15179d570ebac320, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224e1b60, 0x15179d570ebac320, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224e1a40, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224e1a40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224e1a40, 0x5c4d11c, 0x10, 0xc4233499b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224e1a40, 0xc424a42858, 0xc4224e1a40, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224e1a40, 0xc424a42858, 0x1f, 0xc42002f630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421036230, 0xc424a42840, 0x6a551e0, 0xc424a5b440, 0xc400000008, 0xc424dcebc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421036230, 0x6a551e0, 0xc424a5b440, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421036230, 0x6a551e0, 0xc424a5b440, 0xc424a42790, 0xc422c3dbc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114801 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42226dbf0, 0x15179d5718d94ae8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42226dbf0, 0x15179d5718d94ae8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42226dbf0, 0x15179d5718d94ae8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42226db90, 0x15179d5718d94ae8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42226da40, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42226da40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42226da40, 0x5c4d11c, 0x10, 0xc423dd39b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42226da40, 0xc422c820c8, 0xc42226da40, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42226da40, 0xc422c820c8, 0x1f, 0xc42002fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f70ee0, 0xc422c820b0, 0x6a551e0, 0xc42141c500, 0xc400000008, 0xc422e308c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f70ee0, 0x6a551e0, 0xc42141c500, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f70ee0, 0x6a551e0, 0xc42141c500, 0xc422c82000, 0xc423340600) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115252 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4210651d0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4210651d0, 0x15179d570dbdaa28, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4210651d0, 0x15179d570dbdaa28, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4210651d0, 0x15179d570dbdaa28, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4210651a0, 0x15179d570dbdaa28, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4210650b0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4210650b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421065080, 0x5c4d11c, 0x10, 0xc42316d2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421065080, 0xc420e75030, 0xc421065080, 0xc4267ed568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421065080, 0xc420e75030, 0xc420b53500, 0xc423f80750, 0xc423a47240) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc421064f60, 0x5c13340, 0xc420e75030, 0x5c13340, 0xc420e75030, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc421064f60, 0x5c13340, 0xc420e75030, 0xc421b1c280, 0xc421b1c2a0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc421064f60, 0x5c13340, 0xc420e75030, 0xc421b1c280, 0xc421b1c2a0, 0x6, 0xc4267ed968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc421064f60, 0xc420e74fc0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424925c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423874a00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423874a00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423874a00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115147 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc424563410, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc424563410, 0x15179d5720d204b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc424563410, 0x15179d5720d204b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc424563410, 0x15179d5720d204b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4245633b0, 0x15179d5720d204b0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc424563350, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422a7d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc424563350, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc424563350, 0x5c4d11c, 0x10, 0xc4231d19b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc424563350, 0xc421f802d8, 0xc424563350, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc424563350, 0xc421f802d8, 0x1f, 0xc420a7c630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422759ab0, 0xc421f802c0, 0x6a551e0, 0xc4241ccb40, 0xc400000008, 0xc424a16640, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422759ab0, 0x6a551e0, 0xc4241ccb40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422759ab0, 0x6a551e0, 0xc4241ccb40, 0xc421f80210, 0xc42353c3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114782 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b52720, 0x15179d5701de3498, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a6700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b52720, 0x15179d5701de3498, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a6700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422b52720, 0x15179d5701de3498, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a6700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422b52630, 0x15179d5701de3498, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a6700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422b525d0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a6700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422b525d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422b525a0, 0x5c4d11c, 0x10, 0xc423fc72c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422b525a0, 0xc422759ea0, 0xc422b525a0, 0xc4206dd568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422b525a0, 0xc422759ea0, 0xc420b53500, 0xc421ac8138, 0xc4241cd5c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4223f3e30, 0x5c13340, 0xc422759ea0, 0x5c13340, 0xc422759ea0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4223f3e30, 0x5c13340, 0xc422759ea0, 0xc4203a59e0, 0xc4203a5a60, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4223f3e30, 0x5c13340, 0xc422759ea0, 0xc4203a59e0, 0xc4203a5a60, 0x6, 0xc4206dd968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4223f3e30, 0xc422759e30, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba2c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422fae000, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422fae000, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422fae000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 113984 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421bf31d0, 0xc4206a4230, 0x7, 0xc423730280, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421bf31d0, 0x7, 0xc423730280, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421bf31d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421bf31d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421bf31d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421bf31d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421ee4e00, 0x6a552a0, 0xc421bf3050, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4243a1d00, 0xc424f79800, 0x0, 0xc4243a1e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424f79800, 0x1, 0xc421bf2f60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42229afc0, 0xc420718000, 0x5, 0x5, 0x15179d5709dced88, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423e76d50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115225 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc421116a80, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421116a80, 0x15179d5721cd0680, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421116a80, 0x15179d5721cd0680, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421116a80, 0x15179d5721cd0680, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4211169c0, 0x15179d5721cd0680, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4211167e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4211167e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4211167e0, 0x5c4d11c, 0x10, 0xc423ee19b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4211167e0, 0xc421b52438, 0xc4211167e0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4211167e0, 0xc421b52438, 0x1f, 0xc420bcc630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4203420e0, 0xc421b52420, 0x6a551e0, 0xc424105500, 0xc400000008, 0xc4231b6c00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4203420e0, 0x6a551e0, 0xc424105500, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4203420e0, 0x6a551e0, 0xc424105500, 0xc421b52370, 0xc4208012c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114256 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc422027da0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422027da0, 0x15179d570e6f1bf0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422027da0, 0x15179d570e6f1bf0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422027da0, 0x15179d570e6f1bf0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422027d40, 0x15179d570e6f1bf0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422027ce0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422027ce0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422027ce0, 0x5c4d11c, 0x10, 0xc4251859b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422027ce0, 0xc42474e7a8, 0xc422027ce0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422027ce0, 0xc42474e7a8, 0x1f, 0xc420a81630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5ce70, 0xc42474e790, 0x6a551e0, 0xc4231b6840, 0xc400000008, 0xc4226c59c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5ce70, 0x6a551e0, 0xc4231b6840, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5ce70, 0x6a551e0, 0xc4231b6840, 0xc42474e580, 0xc420ba50e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114636 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc424596570, 0xc420518e60, 0x7, 0xc420b97ea0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc424596570, 0x7, 0xc420b97ea0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc424596570, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2b100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc424596570, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2b100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc424596570, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2b100, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc424596570, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2b000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc422e06000, 0x6a552a0, 0xc42380b890, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc422e06000, 0x6a552a0, 0xc42380b890) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc42380b230) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc42380b230, 0xc422608060, 0x29, 0x6a652e0, 0xc420358700, 0xc42048cd40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 113673 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42377a990, 0x15179d56fae723c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42377a990, 0x15179d56fae723c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42377a990, 0x15179d56fae723c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42377a8d0, 0x15179d56fae723c0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42377a840, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42377a840, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42377a840, 0x5c4d11c, 0x10, 0xc424e7d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42377a840, 0xc4241eaa68, 0xc42377a840, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42377a840, 0xc4241eaa68, 0x1f, 0xc420895630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212464d0, 0xc4241eaa50, 0x6a551e0, 0xc422ceee00, 0xc400000008, 0xc4226c4140, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212464d0, 0x6a551e0, 0xc422ceee00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212464d0, 0x6a551e0, 0xc422ceee00, 0xc4241ea790, 0xc422226b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115298 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4221e0500, 0x45563f3, 0xc423d231e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4221e0500, 0x5d2c558, 0xc423d23420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420bd5230, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420bd5230, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a438, 0x5b6ec40, 0xc42390ab40, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc422e30040, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a438, 0x5b6ec40, 0xc42390ab40, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc422e30040, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a438, 0x5b6ec40, 0xc42390ab40, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc422e30040, 0x5c6d3d9, 0x21, 0x5c13340, 0xc423a9a438, 0x5b6ec40, 0xc42390ab40, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80908, 0x6a551e0, 0xc422e30040, 0xc423a9a438, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x800c49042) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4200dfea0, 0xc423a9a420, 0x6a551e0, 0xc422e30040, 0xc400000008, 0xc422e30200, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4200dfea0, 0x6a551e0, 0xc422e30040, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4200dfea0, 0x6a551e0, 0xc422e30040, 0xc423a9a2c0, 0xc42390aa80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114313 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423fbf530, 0xc4206a4230, 0x7, 0xc42382ce10, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423fbf530, 0x7, 0xc42382ce10, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423fbf530, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423fbf530, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423fbf530, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423fbf530, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2c540, 0x6a552a0, 0xc423fbf500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420c0f680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420c0f680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420c0f680, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420c0f680, 0xc4235b3928, 0xc422c57c00, 0x0, 0xc4235b3a60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420c0f680, 0xc422c57c00, 0x1, 0xc423fbf4d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420c0f680, 0xc423dbaea0, 0xc420b53b00, 0x5, 0x5, 0x15179d572254dfd8, 0xc400000000, 0xc4235b3b98, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420c0f680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).runAsyncTask(0xc421013680, 0x6a552a0, 0xc420c0f680, 0xc4224e6700, 0xc42426d201, 0xc423d65e90, 0xc42330e240, 0xc42126f9e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:302 +0x1e4 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync(0xc421013680, 0x6a552a0, 0xc420c0f680, 0xc4224e6700, 0xc4222a4480, 0x1, 0x1, 0x1, 0x0, 0xc4232235f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:380 +0x1b5 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420c0f680, 0x15179d56f4cb4fe8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2763 +0xe93 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420c0f680, 0x15179d56f4cb4fe8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420c0f680, 0x15179d56f4cb4fe8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420c0f650, 0x15179d56f4cb4fe8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420c0f5f0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42440c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420c0f5f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420c0f5c0, 0x5c4d11c, 0x10, 0xc4239312c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420c0f5c0, 0xc4210376c0, 0xc420c0f5c0, 0xc424a0d568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420c0f5c0, 0xc4210376c0, 0xc420b53500, 0xc4241cafb0, 0xc424a5bc80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420c0f470, 0x5c13340, 0xc4210376c0, 0x5c13340, 0xc4210376c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420c0f470, 0x5c13340, 0xc4210376c0, 0xc4215e20a0, 0xc4215e20c0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420c0f470, 0x5c13340, 0xc4210376c0, 0xc4215e20a0, 0xc4215e20c0, 0x6, 0xc424a0d968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420c0f470, 0xc421037650, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422b89c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42257a3c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42257a3c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42257a3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114933 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc42387b740, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387b740, 0x15179d571ac97a58, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387b740, 0x15179d571ac97a58, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42387b740, 0x15179d571ac97a58, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42387b560, 0x15179d571ac97a58, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42387b4d0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42387b4d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42387b4d0, 0x5c4d11c, 0x10, 0xc422ab79b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42387b4d0, 0xc422239cf8, 0xc42387b4d0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42387b4d0, 0xc422239cf8, 0x1f, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420b61b20, 0xc422239ce0, 0x6a551e0, 0xc424a16340, 0xc400000008, 0xc4214523c0, 0x297edd20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420b61b20, 0x6a551e0, 0xc424a16340, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420b61b20, 0x6a551e0, 0xc424a16340, 0xc423601ad0, 0xc4200cd980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115217 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42359c120, 0xc420dd3d50, 0xc423cf8b40, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359c120, 0x15179d56f4cb4fe8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4234e3f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359c120, 0x15179d56f4cb4fe8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4234e3f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42359c120, 0x15179d56f4cb4fe8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4234e3f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42359c0f0, 0x15179d56f4cb4fe8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4234e3f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42359c090, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc4234e3f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42359c090, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42359c060, 0x5c4d11c, 0x10, 0xc424cd12c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42359c060, 0xc420dd3ce0, 0xc42359c060, 0xc424aed568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42359c060, 0xc420dd3ce0, 0xc420b53500, 0xc421ac82d8, 0xc42456f340) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc42124c510, 0x5c13340, 0xc420dd3ce0, 0x5c13340, 0xc420dd3ce0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc42124c510, 0x5c13340, 0xc420dd3ce0, 0xc422d4c5e0, 0xc422d4c620, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc42124c510, 0x5c13340, 0xc420dd3ce0, 0xc422d4c5e0, 0xc422d4c620, 0x6, 0xc424aed968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc42124c510, 0xc420dd3c70, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424920c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fa3c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fa3c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fa3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114753 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4239f9a10, 0xc4206a4230, 0x7, 0xc421640e60, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4239f9a10, 0x7, 0xc421640e60, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4239f9a10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4239f9a10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4239f9a10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4239f9a10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42435e540, 0x6a552a0, 0xc4239f99e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4239ebd00, 0xc422a06000, 0x0, 0xc4239ebe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc422a06000, 0x1, 0xc4239f9980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4222a4b40, 0xc421e20c00, 0x5, 0x5, 0x15179d57177317b0, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc422b14840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114553 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423710ea0, 0x15179d571465d3c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423710ea0, 0x15179d571465d3c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423710ea0, 0x15179d571465d3c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423710e70, 0x15179d571465d3c8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423710e10, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423710e10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423710e10, 0x5c4d11c, 0x10, 0xc4239d19b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423710e10, 0xc421112f38, 0xc423710e10, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423710e10, 0xc421112f38, 0x1f, 0xc420a7c630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4237460e0, 0xc421112f20, 0x6a551e0, 0xc422e30a40, 0xc400000008, 0xc422a5abc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4237460e0, 0x6a551e0, 0xc422e30a40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4237460e0, 0x6a551e0, 0xc422e30a40, 0xc420b43d90, 0xc423737980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114261 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4224e1980, 0xc4206a4230, 0x7, 0xc424699360, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4224e1980, 0x7, 0xc424699360, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4224e1980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4224e1980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4224e1980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4224e1980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421f4a8c0, 0x6a552a0, 0xc4224e1950, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4244d3d00, 0xc423eba700, 0x0, 0xc4244d3e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423eba700, 0x1, 0xc4224e18f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424b965a0, 0xc42326d500, 0x5, 0x5, 0x15179d570e6a9f80, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42130e0f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114752 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b147b0, 0x15179d571772dd18, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b147b0, 0x15179d571772dd18, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422b147b0, 0x15179d571772dd18, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422b14750, 0x15179d571772dd18, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422b146c0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422b146c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422b146c0, 0x5c4d11c, 0x10, 0xc42312b9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422b146c0, 0xc4222b2908, 0xc422b146c0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422b146c0, 0xc4222b2908, 0x1f, 0xc42072de30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420dd3810, 0xc4222b28f0, 0x6a551e0, 0xc42456ec00, 0xc400000008, 0xc422e31680, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420dd3810, 0x6a551e0, 0xc42456ec00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420dd3810, 0x6a551e0, 0xc42456ec00, 0xc4222b2840, 0xc421865500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115111 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc422753320, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422753320, 0x15179d571f1a7d28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422753320, 0x15179d571f1a7d28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422753320, 0x15179d571f1a7d28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4227532c0, 0x15179d571f1a7d28, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422753260, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422753260, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422753260, 0x5c4d11c, 0x10, 0xc4247139b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422753260, 0xc42109c018, 0xc422753260, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422753260, 0xc42109c018, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4226673b0, 0xc42109c000, 0x6a551e0, 0xc42335bb80, 0xc400000008, 0xc422a5b6c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4226673b0, 0x6a551e0, 0xc42335bb80, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4226673b0, 0x6a551e0, 0xc42335bb80, 0xc421f81e40, 0xc421a1c300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114513 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42124d500, 0xc4206a4230, 0x7, 0xc4217c6140, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42124d500, 0x7, 0xc4217c6140, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42124d500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120c300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42124d500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120c300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42124d500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120c300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42124d500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120c000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc421ace000, 0x6a552a0, 0xc422027920, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc421ace000, 0x6a552a0, 0xc422027920) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc4220278f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc4220278f0, 0xc4210eefc0, 0x29, 0x6a652e0, 0xc420b88320, 0xc422171f50) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 115149 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc4219f61b0, 0xc420518e60, 0x7, 0xc423080a50, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc4219f61b0, 0x7, 0xc423080a50, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc4219f61b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc4219f61b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc4219f61b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc4219f61b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc421f4a1c0, 0x6a552a0, 0xc4239f8e70, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc421f4a1c0, 0x6a552a0, 0xc4239f8e70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc42226c540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc42226c540, 0xc4224d2c30, 0x29, 0x6a652e0, 0xc420358700, 0xc4203b2b70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 113436 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc424550ae0, 0xc4206a4230, 0x7, 0xc423cdf6d0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc424550ae0, 0x7, 0xc423cdf6d0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc424550ae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc424550ae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc424550ae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc424550ae0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42435f340, 0x6a552a0, 0xc424550ab0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4245a0f78, 0xc421e30a00, 0x2, 0xc423659420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421e30a00, 0x3, 0xc423659400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4222d0000, 0x32, 0x32, 0x15179d56f8ace748, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4222d0000, 0x32, 0x32, 0x15179d56f8ace748, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4223d41e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114994 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4211e1320, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4211e1320, 0x15179d5708a1fc88, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a0600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4211e1320, 0x15179d5708a1fc88, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a0600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4211e1320, 0x15179d5708a1fc88, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a0600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4211e12f0, 0x15179d5708a1fc88, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a0600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4211e1230, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a0600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4211e1230, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4211e1230, 0x5c4d11c, 0x10, 0xc4242759b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4211e1230, 0xc422f13778, 0xc4211e1230, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4211e1230, 0xc422f13778, 0x1f, 0xc420a83e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4210442a0, 0xc422f13760, 0x6a551e0, 0xc423911ec0, 0xc400000008, 0xc42160ff00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4210442a0, 0x6a551e0, 0xc423911ec0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4210442a0, 0x6a551e0, 0xc423911ec0, 0xc422f12e70, 0xc423535bc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114721 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42275be60, 0x15179d5718fdedd0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42275be60, 0x15179d5718fdedd0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42275be60, 0x15179d5718fdedd0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42275be30, 0x15179d5718fdedd0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42275bda0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42275bda0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42275bda0, 0x5c4d11c, 0x10, 0xc4240b59b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42275bda0, 0xc42276a178, 0xc42275bda0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42275bda0, 0xc42276a178, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42299e070, 0xc42276a160, 0x6a551e0, 0xc42456f200, 0xc400000008, 0xc424dcf4c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42299e070, 0x6a551e0, 0xc42456f200, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42299e070, 0x6a551e0, 0xc42456f200, 0xc421a65970, 0xc423ad6a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114778 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4223f2480, 0xc422758770, 0xc4226bb620, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4223f2480, 0x15179d5719950968, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4223f2480, 0x15179d5719950968, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4223f2480, 0x15179d5719950968, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4223f2450, 0x15179d5719950968, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4223f23f0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4223f23f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4223f23f0, 0x5c4d11c, 0x10, 0xc421fbf9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4223f23f0, 0xc42473ad28, 0xc4223f23f0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4223f23f0, 0xc42473ad28, 0x1f, 0xc420a83e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422758700, 0xc42473ad10, 0x6a551e0, 0xc4241ccc00, 0xc400000008, 0xc423910cc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422758700, 0x6a551e0, 0xc4241ccc00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422758700, 0x6a551e0, 0xc4241ccc00, 0xc42473abb0, 0xc4226bb5c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114672 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4216482d0, 0x15179d5716caf0d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4216482d0, 0x15179d5716caf0d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4216482d0, 0x15179d5716caf0d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4216482a0, 0x15179d5716caf0d0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421648240, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421648240, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421648240, 0x5c4d11c, 0x10, 0xc4233fb9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421648240, 0xc422b66b18, 0xc421648240, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421648240, 0xc422b66b18, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420177dc0, 0xc422b66b00, 0x6a551e0, 0xc423315d40, 0xc400000008, 0xc4220a9140, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420177dc0, 0x6a551e0, 0xc423315d40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420177dc0, 0x6a551e0, 0xc423315d40, 0xc422b668f0, 0xc423a09860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114453 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421684810, 0x15179d571164d098, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421684810, 0x15179d571164d098, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421684810, 0x15179d571164d098, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4219aa840, 0x15179d571164d098, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4219aa7e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4219aa7e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4219aa7e0, 0x5c4d11c, 0x10, 0xc42459f9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4219aa7e0, 0xc422c83568, 0xc4219aa7e0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4219aa7e0, 0xc422c83568, 0x1f, 0xc420033630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4233d3d50, 0xc422c83550, 0x6a551e0, 0xc42160ef00, 0xc400000008, 0xc4238bfc80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4233d3d50, 0x6a551e0, 0xc42160ef00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4233d3d50, 0x6a551e0, 0xc42160ef00, 0xc422c834a0, 0xc420e87860) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114663 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc424596db0, 0xc420518e60, 0x7, 0xc4234e6730, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc424596db0, 0x7, 0xc4234e6730, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc424596db0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2b500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc424596db0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2b500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc424596db0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2b500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc424596db0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2b400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc4249cf6c0, 0x6a552a0, 0xc42379be30, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc4249cf6c0, 0x6a552a0, 0xc42379be30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc42379b7d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc42379b7d0, 0xc4210083c0, 0x29, 0x6a652e0, 0xc420358700, 0xc42164fee0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 111392 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421648900, 0xc4206a4230, 0x7, 0xc421337680, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421648900, 0x7, 0xc421337680, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421648900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421648900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421648900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421648900, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422690e00, 0x6a552a0, 0xc4216488d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4233f2f78, 0xc421b46000, 0x3, 0xc4241042b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421b46000, 0x3, 0xc424104280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423e84000, 0x3a, 0x3a, 0x15179d56c918e928, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423e84000, 0x3a, 0x3a, 0x15179d56c918e928, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc422d997a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115031 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4224e0de0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0de0, 0x15179d571e883bc0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0de0, 0x15179d571e883bc0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224e0de0, 0x15179d571e883bc0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224e0d50, 0x15179d571e883bc0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224e0c60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224e0c60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224e0c60, 0x5c4d11c, 0x10, 0xc422ec39b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224e0c60, 0xc423a9a6f8, 0xc4224e0c60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224e0c60, 0xc423a9a6f8, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4202da9a0, 0xc423a9a6e0, 0x6a551e0, 0xc423e25340, 0xc400000008, 0xc4238beb40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4202da9a0, 0x6a551e0, 0xc423e25340, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4202da9a0, 0x6a551e0, 0xc423e25340, 0xc423a9a630, 0xc422324a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115070 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc420de37c0, 0x45563f3, 0xc423d231e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc420de37c0, 0x5d2c558, 0xc423d23420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4213a5fb0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4213a5fb0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c74d28, 0x5b6ec40, 0xc4234f3680, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc423a46d00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c74d28, 0x5b6ec40, 0xc4234f3680, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc423a46d00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c74d28, 0x5b6ec40, 0xc4234f3680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc423a46d00, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c74d28, 0x5b6ec40, 0xc4234f3680, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f9b18, 0x6a551e0, 0xc423a46d00, 0xc422c74d28, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x9378ff8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423939b90, 0xc422c74d10, 0x6a551e0, 0xc423a46d00, 0xc400000008, 0xc42378c340, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423939b90, 0x6a551e0, 0xc423a46d00, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423939b90, 0x6a551e0, 0xc423a46d00, 0xc422c74c60, 0xc4234f3620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113119 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4212e38c0, 0xc4206a4230, 0x7, 0xc422672e10, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4212e38c0, 0x7, 0xc422672e10, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4212e38c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4212e38c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4212e38c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4212e38c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e08c40, 0x6a552a0, 0xc4212e3860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424cccf78, 0xc421e2ee00, 0x2, 0xc42439d060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421e2ee00, 0x3, 0xc42439d040) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc425180a00, 0x3a, 0x3a, 0x15179d56ecf47218, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc425180a00, 0x3a, 0x3a, 0x15179d56ecf47218, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc424550f30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115034 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4218bd620, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bd620, 0x15179d5709108180, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bd620, 0x15179d5709108180, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4218bd620, 0x15179d5709108180, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4218bd5f0, 0x15179d5709108180, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4218bd560, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421f01f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4218bd560, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4218bd530, 0x5c4d11c, 0x10, 0xc423b772c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4218bd530, 0xc4202dbf80, 0xc4218bd530, 0xc423cc3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4218bd530, 0xc4202dbf80, 0xc420b53500, 0xc423f808b8, 0xc423e25b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4218bd020, 0x5c13340, 0xc4202dbf80, 0x5c13340, 0xc4202dbf80, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4218bd020, 0x5c13340, 0xc4202dbf80, 0xc424bd6380, 0xc424bd63a0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4218bd020, 0x5c13340, 0xc4202dbf80, 0xc424bd6380, 0xc424bd63a0, 0x6, 0xc423cc3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4218bd020, 0xc4202dbe30, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc421925c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422a7b540, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422a7b540, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc422a7b540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114555 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420f65440, 0x15179d5713d98da0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4210fea00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420f65440, 0x15179d5713d98da0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4210fea00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420f65440, 0x15179d5713d98da0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4210fea00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420f65410, 0x15179d5713d98da0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4210fea00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420f653b0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4210fea00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420f653b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420f65350, 0x5c4d11c, 0x10, 0xc423ba32c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420f65350, 0xc423746fc0, 0xc420f65350, 0xc421fbf568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420f65350, 0xc423746fc0, 0xc420b53500, 0xc4241ca410, 0xc422e30f40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc420f651d0, 0x5c13340, 0xc423746fc0, 0x5c13340, 0xc423746fc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc420f651d0, 0x5c13340, 0xc423746fc0, 0xc422b18340, 0xc422b18360, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc420f651d0, 0x5c13340, 0xc423746fc0, 0xc422b18340, 0xc422b18360, 0x6, 0xc421fbf968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc420f651d0, 0xc423746f50, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc420dc5c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422aa17c0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422aa17c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422aa17c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114191 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4236858f0, 0xc4206a4230, 0x7, 0xc420b04aa0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4236858f0, 0x7, 0xc420b04aa0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4236858f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4236858f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4236858f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4236858f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4245ae540, 0x6a552a0, 0xc4236858c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423bc5d00, 0xc425641100, 0x0, 0xc423bc5e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc425641100, 0x1, 0xc423685830) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4216046c0, 0xc420c52600, 0x5, 0x5, 0x15179d570cb594b0, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423919cb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115119 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4225b4390, 0xc4206a4230, 0x7, 0xc422286c80, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4225b4390, 0x7, 0xc422286c80, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4225b4390, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4225b4390, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4225b4390, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4225b4390, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4249ce1c0, 0x6a552a0, 0xc4225b4330, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42338af78, 0xc424f76000, 0x0, 0xc421228210) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424f76000, 0x1, 0xc421228210) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4234e0000, 0x36, 0x36, 0x15179d5720675fe8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4234e0000, 0x36, 0x36, 0x15179d5720675fe8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4222cc060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114727 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422fafb80, 0x45563f3, 0xc4233b91e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422fafb80, 0x5d2c558, 0xc4233b9420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4224f33e0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4224f33e0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42473a648, 0x5b6ec40, 0xc423c202a0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424733880, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42473a648, 0x5b6ec40, 0xc423c202a0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424733880, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42473a648, 0x5b6ec40, 0xc423c202a0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424733880, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42473a648, 0x5b6ec40, 0xc423c202a0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac8f98, 0x6a551e0, 0xc424733880, 0xc42473a648, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7f7a6925f) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420d09c00, 0xc42473a630, 0x6a551e0, 0xc424733880, 0xc400000008, 0xc4232a4640, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420d09c00, 0x6a551e0, 0xc424733880, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420d09c00, 0x6a551e0, 0xc424733880, 0xc42473a160, 0xc423c20240) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114734 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc42124dc80, 0xc420518e60, 0x7, 0xc4217c6500, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc42124dc80, 0x7, 0xc4217c6500, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc42124dc80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc42124dc80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc42124dc80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120c800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc42124dc80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc42120c700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc42293aa80, 0x6a552a0, 0xc4223f2f30, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc42293aa80, 0x6a552a0, 0xc4223f2f30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc424000210) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc424000210, 0xc4224d2480, 0x29, 0x6a652e0, 0xc420358700, 0xc4203b21e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 112265 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc424551860, 0xc4206a4230, 0x7, 0xc422672e60, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc424551860, 0x7, 0xc422672e60, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc424551860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc424551860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc424551860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc424551860, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4237ce540, 0x6a552a0, 0xc424551830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42518af78, 0xc422c5f100, 0x5, 0xc422d9f850) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc422c5f100, 0x5, 0xc422d9f800) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030f900, 0x3e, 0x3e, 0x15179d56d9f1c300, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030f900, 0x3e, 0x3e, 0x15179d56d9f1c300, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42139a990) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114814 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc42473d180, 0x45563f3, 0xc422c951e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc42473d180, 0x5d2c558, 0xc422c95420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4225b5dd0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4225b5dd0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c83358, 0x5b6ec40, 0xc4219e5aa0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424264540, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c83358, 0x5b6ec40, 0xc4219e5aa0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424264540, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c83358, 0x5b6ec40, 0xc4219e5aa0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424264540, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422c83358, 0x5b6ec40, 0xc4219e5aa0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80500, 0x6a551e0, 0xc424264540, 0xc422c83358, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0xca93ac0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421a69d50, 0xc422c83340, 0x6a551e0, 0xc424264540, 0xc400000008, 0xc4220b9c80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421a69d50, 0x6a551e0, 0xc424264540, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421a69d50, 0x6a551e0, 0xc424264540, 0xc422c83290, 0xc4219e5a40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114787 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc423009f20, 0xc4202d8b90, 0x7, 0xc4224f1db0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc423009f20, 0x7, 0xc4224f1db0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc423009f20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200af700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc423009f20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200af700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc423009f20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200af700, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc423009f20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200af500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc423072a80, 0x6a552a0, 0xc424563110, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc423072a80, 0x6a552a0, 0xc424563110) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc424562c60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc424562c60, 0xc4242082d0, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc4203c6320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114609 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc424596180, 0xc420518e60, 0x7, 0xc420b97a90, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc424596180, 0x7, 0xc420b97a90, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc424596180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2ad00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc424596180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2ad00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc424596180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2ad00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc424596180, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421e2ac00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc421ee48c0, 0x6a552a0, 0xc423919b60, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc421ee48c0, 0x6a552a0, 0xc423919b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc4239194a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc4239194a0, 0xc4201e1b00, 0x29, 0x6a652e0, 0xc420358700, 0xc42164faf0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 115220 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4211ea140, 0x45563f3, 0xc424b151e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4211ea140, 0x5d2c558, 0xc424b15420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4223d42d0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4223d42d0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422457358, 0x5b6ec40, 0xc423d62180, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424104940, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422457358, 0x5b6ec40, 0xc423d62180, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424104940, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422457358, 0x5b6ec40, 0xc423d62180, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424104940, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422457358, 0x5b6ec40, 0xc423d62180, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f80428, 0x6a551e0, 0xc424104940, 0xc422457358, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x80be69c72) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212e62a0, 0xc422457340, 0x6a551e0, 0xc424104940, 0xc400000008, 0xc422281240, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212e62a0, 0x6a551e0, 0xc424104940, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212e62a0, 0x6a551e0, 0xc424104940, 0xc4224571e0, 0xc423d62000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114742 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4224f54d0, 0xc4206a4230, 0x7, 0xc423a4e870, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4224f54d0, 0x7, 0xc423a4e870, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4224f54d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4224f54d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4224f54d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4224f54d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42420b180, 0x6a552a0, 0xc4224f54a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42185fd00, 0xc424dc3100, 0x0, 0xc42185fe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424dc3100, 0x1, 0xc4224f5440) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423019320, 0xc420718000, 0x5, 0x5, 0x15179d5716f690c8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4223f2060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114843 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224f27e0, 0x15179d5700f19a48, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b32200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224f27e0, 0x15179d5700f19a48, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b32200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224f27e0, 0x15179d5700f19a48, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b32200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224f27b0, 0x15179d5700f19a48, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b32200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224f2750, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b32200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224f2750, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224f2720, 0x5c4d11c, 0x10, 0xc422bd32c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224f2720, 0xc4225a5880, 0xc4224f2720, 0xc423169568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224f2720, 0xc4225a5880, 0xc420b53500, 0xc4204f8a28, 0xc423ef9c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4224f2390, 0x5c13340, 0xc4225a5880, 0x5c13340, 0xc4225a5880, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4224f2390, 0x5c13340, 0xc4225a5880, 0xc4203e50a0, 0xc4203e50c0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4224f2390, 0x5c13340, 0xc4225a5880, 0xc4203e50a0, 0xc4203e50c0, 0x6, 0xc423169968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4224f2390, 0xc4225a5810, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba4c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423bacb40, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423bacb40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423bacb40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114222 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc421181cb0, 0xc4202d8b90, 0x7, 0xc42350a230, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc421181cb0, 0x7, 0xc42350a230, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc421181cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4234e3d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc421181cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4234e3d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc421181cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4234e3d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc421181cb0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4234e3c00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc422e076c0, 0x6a552a0, 0xc423710270, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc422e076c0, 0x6a552a0, 0xc423710270) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc4228d3cb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc4228d3cb0, 0xc4201e06c0, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc4203b2950) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114416 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d64c90, 0x15179d57116e8cc8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422246d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d64c90, 0x15179d57116e8cc8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422246d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423d64c90, 0x15179d57116e8cc8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422246d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423d64c60, 0x15179d57116e8cc8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422246d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423d64c00, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422246d00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423d64c00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423d64bd0, 0x5c4d11c, 0x10, 0xc42411d2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423d64bd0, 0xc420f71dc0, 0xc423d64bd0, 0xc424695568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423d64bd0, 0xc420f71dc0, 0xc420b53500, 0xc423f801f0, 0xc424265580) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc423d64a80, 0x5c13340, 0xc420f71dc0, 0x5c13340, 0xc420f71dc0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc423d64a80, 0x5c13340, 0xc420f71dc0, 0xc422c2d680, 0xc422c2d6a0, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc423d64a80, 0x5c13340, 0xc420f71dc0, 0xc422c2d680, 0xc422c2d6a0, 0x6, 0xc424695968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc423d64a80, 0xc420f71d50, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc424921c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424b3c500, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424b3c500, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424b3c500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114585 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc420d24270, 0xc4206a4230, 0x7, 0xc42484eff0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc420d24270, 0x7, 0xc42484eff0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc420d24270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc420d24270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc420d24270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc420d24270, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c3bdc0, 0x6a552a0, 0xc420d24240, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421aafd00, 0xc42362a000, 0x0, 0xc421aafe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42362a000, 0x1, 0xc420d241e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423dbbc20, 0xc422718c00, 0x5, 0x5, 0x15179d57149d5e88, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423f5cab0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115268 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42124d890, 0xc42311c930, 0xc423e0bb00, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124d890, 0x15179d56fad1e640, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42124d890, 0x15179d56fad1e640, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42124d890, 0x15179d56fad1e640, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42124d7a0, 0x15179d56fad1e640, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42124d740, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120c400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42124d740, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42124d740, 0x5c4d11c, 0x10, 0xc4237c79b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42124d740, 0xc4231d3618, 0xc42124d740, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42124d740, 0xc4231d3618, 0x1f, 0xc420a81630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42311c8c0, 0xc4231d3600, 0x6a551e0, 0xc422495040, 0xc400000008, 0xc4226c4940, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42311c8c0, 0x6a551e0, 0xc422495040, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42311c8c0, 0x6a551e0, 0xc422495040, 0xc4231d3340, 0xc423e0ba40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114956 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421a3b7a0, 0xc4206a4230, 0x7, 0xc4230803c0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421a3b7a0, 0x7, 0xc4230803c0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421a3b7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421a3b7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421a3b7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421a3b7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200afc00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc4230736c0, 0x6a552a0, 0xc421a3a7e0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc4230736c0, 0x6a552a0, 0xc421a3a7e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc4219f7d10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc4219f7d10, 0xc4204501e0, 0x29, 0x6a652e0, 0xc420b88320, 0xc420047e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114528 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42130d9b0, 0x15179d5713952200, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42130d9b0, 0x15179d5713952200, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42130d9b0, 0x15179d5713952200, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42130d980, 0x15179d5713952200, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42130d8c0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42130d8c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42130d8c0, 0x5c4d11c, 0x10, 0xc4243d39b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42130d8c0, 0xc42474fae8, 0xc42130d8c0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42130d8c0, 0xc42474fae8, 0x1f, 0xc420a7ee30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421081e30, 0xc42474fad0, 0x6a551e0, 0xc422a5a180, 0xc400000008, 0xc423307000, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421081e30, 0x6a551e0, 0xc422a5a180, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421081e30, 0x6a551e0, 0xc422a5a180, 0xc42474fa20, 0xc423cf9320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114002 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4221062a0, 0x15179d57053429e0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4221062a0, 0x15179d57053429e0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4221062a0, 0x15179d57053429e0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422106240, 0x15179d57053429e0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4221061e0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4221061e0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4221061e0, 0x5c4d11c, 0x10, 0xc42354d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4221061e0, 0xc4236b20c8, 0xc4221061e0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4221061e0, 0xc4236b20c8, 0x1f, 0xc420a83e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4223262a0, 0xc4236b20b0, 0x6a551e0, 0xc4236582c0, 0xc400000008, 0xc42397cb00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4223262a0, 0x6a551e0, 0xc4236582c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4223262a0, 0x6a551e0, 0xc4236582c0, 0xc4236b2000, 0xc423d85ec0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 109944 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423f5ce70, 0xc4206a4230, 0x7, 0xc420dacff0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423f5ce70, 0x7, 0xc420dacff0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423f5ce70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423f5ce70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423f5ce70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423f5ce70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c26700, 0x6a552a0, 0xc423f5ce40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423138f78, 0xc421949c00, 0x4, 0xc423821740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc421949c00, 0x5, 0xc423821700) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x3a, 0x3a, 0x15179d56ad511ff8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x3a, 0x3a, 0x15179d56ad511ff8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423f5cc30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115037 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4218bdad0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bdad0, 0x15179d571ec0b8b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bdad0, 0x15179d571ec0b8b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4218bdad0, 0x15179d571ec0b8b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4218bdaa0, 0x15179d571ec0b8b0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4218bda40, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4218bda40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4218bda40, 0x5c4d11c, 0x10, 0xc423b779b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4218bda40, 0xc423a9b4b8, 0xc4218bda40, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4218bda40, 0xc423a9b4b8, 0x1f, 0xc420899e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420cea770, 0xc423a9b4a0, 0x6a551e0, 0xc423e25c40, 0xc400000008, 0xc42378df00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420cea770, 0x6a551e0, 0xc423e25c40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420cea770, 0x6a551e0, 0xc423e25c40, 0xc423a9b340, 0xc4236daf00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114378 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc42359cc30, 0xc4202d8b90, 0x7, 0xc424782f00, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc42359cc30, 0x7, 0xc424782f00, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc42359cc30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc42359cc30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc42359cc30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc42359cc30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420720f00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc4224d48c0, 0x6a552a0, 0xc4223d5a10, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc4224d48c0, 0x6a552a0, 0xc4223d5a10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc4223d5740) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc4223d5740, 0xc4217e8660, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc4203947b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 115270 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc4224e0db0, 0xc42311d260, 0xc4238eecc0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0db0, 0x15179d5705e10f98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4224e0db0, 0x15179d5705e10f98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4224e0db0, 0x15179d5705e10f98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224e0b70, 0x15179d5705e10f98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224e0b10, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc42120d500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224e0b10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224e0ae0, 0x5c4d11c, 0x10, 0xc42313d2c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224e0ae0, 0xc42311d1f0, 0xc4224e0ae0, 0xc425463568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224e0ae0, 0xc42311d1f0, 0xc420b53500, 0xc4241ca648, 0xc422a5a280) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4224e09f0, 0x5c13340, 0xc42311d1f0, 0x5c13340, 0xc42311d1f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4224e09f0, 0x5c13340, 0xc42311d1f0, 0xc422e27f20, 0xc422e27f40, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4224e09f0, 0x5c13340, 0xc42311d1f0, 0xc422e27f20, 0xc422e27f40, 0x6, 0xc425463968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4224e09f0, 0xc42311d180, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc421b04c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fb400, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fb400, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc4237fb400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 115246 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc421a3bf80, 0xc420e5cf50, 0xc4237b0c60, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a3bf80, 0x15179d570bdb0778, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a3bf80, 0x15179d570bdb0778, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421a3bf80, 0x15179d570bdb0778, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421a3bf50, 0x15179d570bdb0778, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421a3bef0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0xc423524400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421a3bef0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421a3bef0, 0x5c4d11c, 0x10, 0xc4232419b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421a3bef0, 0xc42474fb98, 0xc421a3bef0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421a3bef0, 0xc42474fb98, 0x1f, 0xc42002fe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5cd20, 0xc42474fb80, 0x6a551e0, 0xc42397d580, 0xc400000008, 0xc42397d700, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5cd20, 0x6a551e0, 0xc42397d580, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5cd20, 0x6a551e0, 0xc42397d580, 0xc42474fa20, 0xc4237b0a20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114763 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc423009830, 0xc4202d8b90, 0x7, 0xc4224f1900, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc423009830, 0x7, 0xc4224f1900, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc423009830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200af400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc423009830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200af400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc423009830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200af400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc423009830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4200af300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc423d31c00, 0x6a552a0, 0xc420f6d8f0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc423d31c00, 0x6a552a0, 0xc420f6d8f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc420f6d200) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc420f6d200, 0xc420187320, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc42032ea80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114988 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423710ff0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423710ff0, 0x15179d571bbd8eb8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42272c300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423710ff0, 0x15179d571bbd8eb8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42272c300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423710ff0, 0x15179d571bbd8eb8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42272c300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423710fc0, 0x15179d571bbd8eb8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42272c300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423710f60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42272c300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423710f60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423710f60, 0x5c4d11c, 0x10, 0xc423df39b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423710f60, 0xc423e2afe8, 0xc423710f60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423710f60, 0xc423e2afe8, 0x1f, 0xc420a83e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420be5ab0, 0xc423e2afd0, 0x6a551e0, 0xc422a5af00, 0xc400000008, 0xc4233152c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420be5ab0, 0x6a551e0, 0xc422a5af00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420be5ab0, 0x6a551e0, 0xc422a5af00, 0xc423e2af20, 0xc423408d20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113360 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420732fc0, 0x15179d56f6f26608, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420732fc0, 0x15179d56f6f26608, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420732fc0, 0x15179d56f6f26608, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420732f60, 0x15179d56f6f26608, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420732f00, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420732f00, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420732f00, 0x5c4d11c, 0x10, 0xc4233559b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420732f00, 0xc4231d2d28, 0xc420732f00, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420732f00, 0xc4231d2d28, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420e5c460, 0xc4231d2d10, 0x6a551e0, 0xc4231b6380, 0xc400000008, 0xc424918940, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420e5c460, 0x6a551e0, 0xc4231b6380, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420e5c460, 0x6a551e0, 0xc4231b6380, 0xc4231d2b00, 0xc421ed5620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114957 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc421a3aa50, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a3aa50, 0x15179d571d075c68, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421a3aa50, 0x15179d571d075c68, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421a3aa50, 0x15179d571d075c68, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc421a3aa20, 0x15179d571d075c68, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421a3a9c0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421a3a9c0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421a3a9c0, 0x5c4d11c, 0x10, 0xc4239039b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421a3a9c0, 0xc423b300c8, 0xc421a3a9c0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421a3a9c0, 0xc423b300c8, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4225284d0, 0xc423b300b0, 0x6a551e0, 0xc42397db80, 0xc400000008, 0xc424918300, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4225284d0, 0x6a551e0, 0xc42397db80, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4225284d0, 0x6a551e0, 0xc42397db80, 0xc420c75b80, 0xc4238ef2c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113554 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423293530, 0x15179d56f93dc538, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423293530, 0x15179d56f93dc538, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423293530, 0x15179d56f93dc538, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4232934d0, 0x15179d56f93dc538, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423293440, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423293440, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423293440, 0x5c4d11c, 0x10, 0xc4238099b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423293440, 0xc423d7ef38, 0xc423293440, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423293440, 0xc423d7ef38, 0x1f, 0xc42072de30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42299f570, 0xc423d7ef20, 0x6a551e0, 0xc42439d840, 0xc400000008, 0xc4238bfd00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42299f570, 0x6a551e0, 0xc42439d840, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42299f570, 0x6a551e0, 0xc42439d840, 0xc423d7edc0, 0xc421fc7b00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 108767 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42166a6c0, 0xc4206a4230, 0x7, 0xc423c40910, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42166a6c0, 0x7, 0xc423c40910, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42166a6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42166a6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42166a6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42166a6c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42209a000, 0x6a552a0, 0xc42166a690, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a55260, 0xc4234f3ce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a55260, 0xc4234f3ce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a55260, 0xc4234f3ce0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a55260, 0xc4234f3ce0, 0xc423eefa80, 0xc42546d100, 0x6cae460, 0x6a55260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a55260, 0xc4234f3ce0, 0xc42546d100, 0xc4234f3ce0, 0xc4211a3000) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424ead200, 0xc, 0x10, 0x15179d56949b0101, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:625 +0x7a2 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42195e800, 0x45, 0x45, 0x15179d56949b78c8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:365 +0x196 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420f64480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 113502 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421116e10, 0xc4206a4230, 0x7, 0xc4218dd270, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421116e10, 0x7, 0xc4218dd270, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421116e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421116e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421116e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421116e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423645dc0, 0x6a552a0, 0xc421116de0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc420d01d00, 0xc42235d800, 0x0, 0xc420d01e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc42235d800, 0x1, 0xc421116d80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423a9db00, 0xc421bab800, 0x5, 0x5, 0x15179d56f8b0ac20, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc424550d20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114403 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4224e0210, 0xc4206a4230, 0x7, 0xc422791bd0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4224e0210, 0x7, 0xc422791bd0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4224e0210, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4224e0210, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4224e0210, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4224e0210, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4237cf6c0, 0x6a552a0, 0xc4224e01e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc422337d00, 0xc424f6ee00, 0x0, 0xc422337e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424f6ee00, 0x1, 0xc4224e0120) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42399dd40, 0xc421bc0600, 0x5, 0x5, 0x15179d57109320e8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42248f0b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114701 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b21f50, 0x15179d5717007408, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422b21f50, 0x15179d5717007408, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422b21f50, 0x15179d5717007408, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422b21ef0, 0x15179d5717007408, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422b21e90, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422b21e90, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422b21e90, 0x5c4d11c, 0x10, 0xc424a2f9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422b21e90, 0xc421633098, 0xc422b21e90, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422b21e90, 0xc421633098, 0x1f, 0xc420bcbe30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420b335e0, 0xc421633080, 0x6a551e0, 0xc42378de00, 0xc400000008, 0xc422e31c00, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420b335e0, 0x6a551e0, 0xc42378de00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420b335e0, 0x6a551e0, 0xc42378de00, 0xc421632f20, 0xc423c5ac00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113863 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4223ee9f0, 0xc4206a4230, 0x7, 0xc421b55810, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4223ee9f0, 0x7, 0xc421b55810, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4223ee9f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4223ee9f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4223ee9f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4223ee9f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42420ae00, 0x6a552a0, 0xc4223ee9c0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42409fd00, 0xc4225f7800, 0x0, 0xc42409fe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4225f7800, 0x1, 0xc4223ee960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4222a4c60, 0xc422740c00, 0x5, 0x5, 0x15179d5701a83848, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4225d4660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114488 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc420c0eff0, 0xc422307f10, 0xc420800a80, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420c0eff0, 0x15179d571279ab98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420c0eff0, 0x15179d571279ab98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420c0eff0, 0x15179d571279ab98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420c0ef90, 0x15179d571279ab98, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420c0ef30, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420c0ef30, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420c0ef30, 0x5c4d11c, 0x10, 0xc4216739b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420c0ef30, 0xc42371fcf8, 0xc420c0ef30, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420c0ef30, 0xc42371fcf8, 0x1f, 0xc42072de30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422667f10, 0xc42371fce0, 0x6a551e0, 0xc422494800, 0xc400000008, 0xc4220a9f80, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422667f10, 0x6a551e0, 0xc422494800, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422667f10, 0x6a551e0, 0xc422494800, 0xc42371fb80, 0xc421238420) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114518 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359d410, 0x15179d56f99c9a90, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42359d410, 0x15179d56f99c9a90, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42359d410, 0x15179d56f99c9a90, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42359d3e0, 0x15179d56f99c9a90, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42359d380, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc420db3000, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42359d380, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42359d350, 0x5c4d11c, 0x10, 0xc422dc92c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42359d350, 0xc421036e70, 0xc42359d350, 0xc422cf3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42359d350, 0xc421036e70, 0xc420b53500, 0xc423f80500, 0xc424a5a580) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc42359d170, 0x5c13340, 0xc421036e70, 0x5c13340, 0xc421036e70, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc42359d170, 0x5c13340, 0xc421036e70, 0xc422e27f20, 0xc422e27f40, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc42359d170, 0x5c13340, 0xc421036e70, 0xc422e27f20, 0xc422e27f40, 0x6, 0xc422cf3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc42359d170, 0xc421036e00, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422b89c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424dc4dc0, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424dc4dc0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc424dc4dc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114773 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422a0d050, 0x15179d57170a2480, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc422a0d050, 0x15179d57170a2480, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc422a0d050, 0x15179d57170a2480, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc422a0cff0, 0x15179d57170a2480, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc422a0cf30, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc424605900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc422a0cf30, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc422a0cf00, 0x5c4d11c, 0x10, 0xc424e852c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc422a0cf00, 0xc422758bd0, 0xc422a0cf00, 0xc421fb9568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc422a0cf00, 0xc422758bd0, 0xc420b53500, 0xc421ac9270, 0xc4232a5b00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc422a0ccc0, 0x5c13340, 0xc422758bd0, 0x5c13340, 0xc422758bd0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc422a0ccc0, 0x5c13340, 0xc422758bd0, 0xc4203e5f20, 0xc4203e5f40, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc422a0ccc0, 0x5c13340, 0xc422758bd0, 0xc4203e5f20, 0xc4203e5f40, 0x6, 0xc421fb9968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc422a0ccc0, 0xc422758af0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc4210a5c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42311ba40, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42311ba40, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc42311ba40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114876 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42375fc50, 0xc4206a4230, 0x7, 0xc424199db0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42375fc50, 0x7, 0xc424199db0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42375fc50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42375fc50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42375fc50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42375fc50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421c3afc0, 0x6a552a0, 0xc42375fc20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42591ef78, 0xc423de0000, 0x1, 0xc422432970) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423de0000, 0x2, 0xc422432960) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030d400, 0x3a, 0x3a, 0x15179d571c3527e8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030d400, 0x3a, 0x3a, 0x15179d571c3527e8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42375fb90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 111989 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421a3b620, 0xc4206a4230, 0x7, 0xc422673810, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421a3b620, 0x7, 0xc422673810, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421a3b620, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421a3b620, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421a3b620, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421a3b620, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42293bdc0, 0x6a552a0, 0xc421a3b5f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4237e0f78, 0xc424a73800, 0x2, 0xc424a5a320) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424a73800, 0x3, 0xc424a5a300) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x3a, 0x3a, 0x15179d56d4c18b90, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42030af00, 0x3a, 0x3a, 0x15179d56d4c18b90, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420c0eba0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114517 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc424dc4b40, 0x45563f3, 0xc4245811e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc424dc4b40, 0x5d2c558, 0xc424581420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42359cde0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42359cde0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79618, 0x5b6ec40, 0xc422325f80, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424a5a4c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79618, 0x5b6ec40, 0xc422325f80, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424a5a4c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79618, 0x5b6ec40, 0xc422325f80, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424a5a4c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79618, 0x5b6ec40, 0xc422325f80, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f804f8, 0x6a551e0, 0xc424a5a4c0, 0xc422a79618, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7fc73d608) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421036d20, 0xc422a79600, 0x6a551e0, 0xc424a5a4c0, 0xc400000008, 0xc422cef9c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421036d20, 0x6a551e0, 0xc424a5a4c0, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421036d20, 0x6a551e0, 0xc424a5a4c0, 0xc422a79550, 0xc422325f20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113358 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423fbe720, 0xc4206a4230, 0x7, 0xc42382c640, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423fbe720, 0x7, 0xc42382c640, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423fbe720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423fbe720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423fbe720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423fbe720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e2c380, 0x6a552a0, 0xc423fbe5d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a55260, 0xc42168cf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a55260, 0xc42168cf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a55260, 0xc42168cf60, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a55260, 0xc42168cf60, 0xc421969a80, 0xc422c56000, 0x6cae460, 0x6a55260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a55260, 0xc42168cf60, 0xc422c56000, 0xc42168cf60, 0xc4203db500) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).resolveIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421857d40, 0x2, 0x2, 0x15179d56f68b0101, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:625 +0x7a2 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4218c2000, 0x30, 0x30, 0x15179d56f68b6ea8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:365 +0x196 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420732720) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115197 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423fbea20, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbea20, 0x15179d57225a7970, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423fbea20, 0x15179d57225a7970, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423fbea20, 0x15179d57225a7970, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423fbe930, 0x15179d57225a7970, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423fbe8a0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423fbe8a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423fbe8a0, 0x5c4d11c, 0x10, 0xc4224ad9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423fbe8a0, 0xc42371fb98, 0xc423fbe8a0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423fbe8a0, 0xc42371fb98, 0x1f, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422667730, 0xc42371fb80, 0x6a551e0, 0xc42403bcc0, 0xc400000008, 0xc42403a5c0, 0x30928260) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422667730, 0x6a551e0, 0xc42403bcc0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422667730, 0x6a551e0, 0xc42403bcc0, 0xc42371fa20, 0xc42168cfc0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114711 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4229a1ec0, 0x15179d56fba9b070, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4229a1ec0, 0x15179d56fba9b070, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4229a1ec0, 0x15179d56fba9b070, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4229a1e30, 0x15179d56fba9b070, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4229a1d40, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4229a1d40, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4229a1ce0, 0x5c4d11c, 0x10, 0xc4219f52c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4229a1ce0, 0xc4225eab60, 0xc4229a1ce0, 0xc42552d568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4229a1ce0, 0xc4225eab60, 0xc420b53500, 0xc423f810d8, 0xc42403b0c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4229a1a40, 0x5c13340, 0xc4225eab60, 0x5c13340, 0xc4225eab60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4229a1a40, 0x5c13340, 0xc4225eab60, 0xc4211d8b00, 0xc4211d8b20, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4229a1a40, 0x5c13340, 0xc4225eab60, 0xc4211d8b00, 0xc4211d8b20, 0x6, 0xc42552d968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4229a1a40, 0xc4225eaaf0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422b89c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423506140, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423506140, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc423506140) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114710 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42370f830, 0x15179d56fe4cefb8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42341d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42370f830, 0x15179d56fe4cefb8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42341d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42370f830, 0x15179d56fe4cefb8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42341d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42370f800, 0x15179d56fe4cefb8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42341d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42370f7a0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42341d600, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42370f7a0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42370f770, 0x5c4d11c, 0x10, 0xc422e632c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42370f770, 0xc4202698f0, 0xc42370f770, 0xc424be3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42370f770, 0xc4202698f0, 0xc420b53500, 0xc4241caa10, 0xc42378d240) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4229a15f0, 0x5c13340, 0xc4202698f0, 0x5c13340, 0xc4202698f0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4229a15f0, 0x5c13340, 0xc4202698f0, 0xc421c2b320, 0xc421c2b340, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4229a15f0, 0x5c13340, 0xc4202698f0, 0xc421c2b320, 0xc421c2b340, 0x6, 0xc424be3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4229a15f0, 0xc420269810, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba2c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422fafe00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422fafe00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc421eda7f0, 0xc4205e8420, 0x6a60d60, 0xc4219bc300, 0xc422fafe00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 112897 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423890e40, 0xc4206a4230, 0x7, 0xc423087360, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423890e40, 0x7, 0xc423087360, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423890e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423890e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423890e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423890e40, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42341e000, 0x6a552a0, 0xc423890e10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4208aaf78, 0xc4250f5800, 0x2, 0xc423e243e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4250f5800, 0x3, 0xc423e243c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423afe000, 0x3e, 0x3e, 0x15179d56e7b4d158, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423afe000, 0x3e, 0x3e, 0x15179d56e7b4d158, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420b23c80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114577 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b451a0, 0x15179d571571c1a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423b451a0, 0x15179d571571c1a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423b451a0, 0x15179d571571c1a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423b45140, 0x15179d571571c1a0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423b44f60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423b44f60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423b44f60, 0x5c4d11c, 0x10, 0xc42433f9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423b44f60, 0xc4212114b8, 0xc423b44f60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423b44f60, 0xc4212114b8, 0x1f, 0xc420a7ee30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421247570, 0xc4212114a0, 0x6a551e0, 0xc4238be800, 0xc400000008, 0xc4231b7780, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421247570, 0x6a551e0, 0xc4238be800, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421247570, 0x6a551e0, 0xc4238be800, 0xc423c1bce0, 0xc4228f35c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115124 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4216845d0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4216845d0, 0x15179d571ed77118, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4216845d0, 0x15179d571ed77118, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4216845d0, 0x15179d571ed77118, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4216845a0, 0x15179d571ed77118, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421684540, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421684540, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421684540, 0x5c4d11c, 0x10, 0xc4222779b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421684540, 0xc422c75cf8, 0xc421684540, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421684540, 0xc422c75cf8, 0x1f, 0xc420895630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212e77a0, 0xc422c75ce0, 0x6a551e0, 0xc42160e480, 0xc400000008, 0xc421452bc0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212e77a0, 0x6a551e0, 0xc42160e480, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212e77a0, 0x6a551e0, 0xc42160e480, 0xc422c75b80, 0xc422b68840) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114668 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc42387ae10, 0xc420176d90, 0xc423a08ba0, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387ae10, 0x15179d5716aa8408, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387ae10, 0x15179d5716aa8408, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42387ae10, 0x15179d5716aa8408, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42387acf0, 0x15179d5716aa8408, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42387ac60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42387ac60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42387ac60, 0x5c4d11c, 0x10, 0xc422e639b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42387ac60, 0xc4235564e8, 0xc42387ac60, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42387ac60, 0xc4235564e8, 0x1f, 0x0, 0x18) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420176d20, 0xc4235564d0, 0x6a551e0, 0xc423315600, 0xc400000008, 0xc423911440, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420176d20, 0x6a551e0, 0xc423315600, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420176d20, 0x6a551e0, 0xc423315600, 0xc42296fce0, 0xc423a08ae0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115107 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc423d65cb0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d65cb0, 0x15179d571e16c4e0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412f200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423d65cb0, 0x15179d571e16c4e0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412f200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423d65cb0, 0x15179d571e16c4e0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412f200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423d65c80, 0x15179d571e16c4e0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412f200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423d65c20, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc42412f200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423d65c20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423d65c20, 0x5c4d11c, 0x10, 0xc4235a59b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423d65c20, 0xc421f81098, 0xc423d65c20, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423d65c20, 0xc421f81098, 0x1f, 0xc42072d630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420343b90, 0xc421f81080, 0x6a551e0, 0xc42335ad40, 0xc400000008, 0xc422e30c00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420343b90, 0x6a551e0, 0xc42335ad40, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420343b90, 0x6a551e0, 0xc42335ad40, 0xc421f80c60, 0xc420b3c1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114863 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc42282cdc0, 0x45563f3, 0xc4248a31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc42282cdc0, 0x5d2c558, 0xc4248a3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc422e598c0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc422e598c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4241c5e58, 0x5b6ec40, 0xc42418e5a0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc421450280, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4241c5e58, 0x5b6ec40, 0xc42418e5a0, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc421450280, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4241c5e58, 0x5b6ec40, 0xc42418e5a0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc421450280, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4241c5e58, 0x5b6ec40, 0xc42418e5a0, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca2d0, 0x6a551e0, 0xc421450280, 0xc4241c5e58, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x4015826) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420d089a0, 0xc4241c5e40, 0x6a551e0, 0xc421450280, 0xc400000008, 0xc42160f180, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420d089a0, 0x6a551e0, 0xc421450280, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420d089a0, 0x6a551e0, 0xc421450280, 0xc4241c5600, 0xc42418e540) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113172 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422b21e30, 0xc4206a4230, 0x7, 0xc4217c6d20, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422b21e30, 0x7, 0xc4217c6d20, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422b21e30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422b21e30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422b21e30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422b21e30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423e09a40, 0x6a552a0, 0xc422b21e00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc4245a2f78, 0xc4202bfc00, 0x1, 0xc42037de30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4202bfc00, 0x2, 0xc42037de20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4218d8000, 0x38, 0x38, 0x15179d56f0cb9998, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4218d8000, 0x38, 0x38, 0x15179d56f0cb9998, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc4231221e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 113612 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc42376c450, 0xc4206a4230, 0x7, 0xc4219ea640, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc42376c450, 0x7, 0xc4219ea640, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc42376c450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc42376c450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc42376c450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc42376c450, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4247c5340, 0x6a552a0, 0xc42376c420, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc420a28f78, 0xc423b89800, 0x6, 0xc423641be0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423b89800, 0x5, 0xc423641b80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x3e, 0x3e, 0x15179d56fa91a120, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc420306500, 0x3e, 0x3e, 0x15179d56fa91a120, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc420733290) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 113983 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4229a04e0, 0xc4206a4230, 0x7, 0xc4239c2460, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4229a04e0, 0x7, 0xc4239c2460, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4229a04e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4229a04e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4229a04e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4229a04e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421a81a40, 0x6a552a0, 0xc4229a04b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424cdef78, 0xc425243800, 0x3, 0xc424919b70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc425243800, 0x3, 0xc424919b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42318c000, 0x3c, 0x3c, 0x15179d5709c84860, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42318c000, 0x3c, 0x3c, 0x15179d5709c84860, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423e76690) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114894 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422c13e00, 0x45563f3, 0xc422c731e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422c13e00, 0x5d2c558, 0xc422c73420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4234079e0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4234079e0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422108438, 0x5b6ec40, 0xc423d75620, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4220a87c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422108438, 0x5b6ec40, 0xc423d75620, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc4220a87c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422108438, 0x5b6ec40, 0xc423d75620, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4220a87c0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422108438, 0x5b6ec40, 0xc423d75620, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac83a0, 0x6a551e0, 0xc4220a87c0, 0xc422108438, 0x0, 0x0, 0x0, 0x1, 0x4014366, 0x9364f88) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42102c380, 0xc422108420, 0x6a551e0, 0xc4220a87c0, 0xc400000008, 0xc424a17600, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42102c380, 0x6a551e0, 0xc4220a87c0, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42102c380, 0x6a551e0, 0xc4220a87c0, 0xc421101550, 0xc423d755c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114877 [select, 1 minutes]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).beginCmds(0xc4224e6700, 0x6a552a0, 0xc421f53290, 0xc423553500, 0xc422981680, 0x1, 0x8, 0xc4201e75f0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2211 +0x660 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421f53290, 0x15179d571c389ab8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2665 +0x12f7 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc421f53290, 0x15179d571c389ab8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc421f53290, 0x15179d571c389ab8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42375fe00, 0x15179d571c389ab8, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42375fda0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42375fda0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42375fda0, 0x5c4d11c, 0x10, 0xc422f4d9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42375fda0, 0xc422c83ae8, 0xc42375fda0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42375fda0, 0xc422c83ae8, 0x1f, 0xc420899e30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc423f516c0, 0xc422c83ad0, 0x6a551e0, 0xc423e247c0, 0xc400000008, 0xc4233070c0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc423f516c0, 0x6a551e0, 0xc423e247c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc423f516c0, 0x6a551e0, 0xc423e247c0, 0xc422c83a20, 0xc4238af620) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114316 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423fbf7a0, 0xc4206a4230, 0x7, 0xc421b54000, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423fbf7a0, 0x7, 0xc421b54000, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423fbf7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423fbf7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423fbf7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423fbf7a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc4237ce8c0, 0x6a552a0, 0xc423fbf770, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421adfd00, 0xc4242a3500, 0x0, 0xc421adfe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4242a3500, 0x1, 0xc423fbf5c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4236f3b00, 0xc4211de600, 0x5, 0x5, 0x15179d570f20b0e0, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423293e90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114584 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423f5c8d0, 0x15179d57144d1f90, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a1500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423f5c8d0, 0x15179d57144d1f90, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a1500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423f5c8d0, 0x15179d57144d1f90, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a1500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423f5c8a0, 0x15179d57144d1f90, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a1500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423f5c840, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc4212a1500, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423f5c840, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423f5c840, 0x5c4d11c, 0x10, 0xc42328f9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423f5c840, 0xc423e28a68, 0xc423f5c840, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423f5c840, 0xc423e28a68, 0x1f, 0xc420bc9630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc42193b180, 0xc423e28a50, 0x6a551e0, 0xc422ceea00, 0xc400000008, 0xc423aa1800, 0x8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc42193b180, 0x6a551e0, 0xc422ceea00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc42193b180, 0x6a551e0, 0xc422ceea00, 0xc423e284d0, 0xc4237d6c00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115135 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc424284780, 0x45563f3, 0xc4241491e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc424284780, 0x5d2c558, 0xc424149420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420c4bad0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420c4bad0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4218b4908, 0x5b6ec40, 0xc42414efc0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42378cec0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4218b4908, 0x5b6ec40, 0xc42414efc0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc42378cec0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4218b4908, 0x5b6ec40, 0xc42414efc0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42378cec0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4218b4908, 0x5b6ec40, 0xc42414efc0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac8070, 0x6a551e0, 0xc42378cec0, 0xc4218b4908, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7dadb182e) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4233d2540, 0xc4218b48f0, 0x6a551e0, 0xc42378cec0, 0xc400000008, 0xc421453400, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4233d2540, 0x6a551e0, 0xc42378cec0, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4233d2540, 0x6a551e0, 0xc42378cec0, 0xc4218b4840, 0xc42414eea0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 113503 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc423293290, 0xc4206a4230, 0x7, 0xc42411eaf0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc423293290, 0x7, 0xc42411eaf0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc423293290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc423293290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc423293290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc423293290, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc42209ae00, 0x6a552a0, 0xc423293140, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424476f78, 0xc422772700, 0x0, 0xc4212284b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc422772700, 0x1, 0xc4212284b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4235d8000, 0x32, 0x32, 0x15179d56f92b1fc8, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc4235d8000, 0x32, 0x32, 0x15179d56f92b1fc8, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc424551530) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 112084 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc4237fe5a0, 0xc4206a4230, 0x7, 0xc422be9770, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc4237fe5a0, 0x7, 0xc422be9770, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc4237fe5a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc4237fe5a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc4237fe5a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc4237fe5a0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc423d30e00, 0x6a552a0, 0xc4237fe570, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc424e30f78, 0xc4221bf800, 0x3, 0xc424a16d70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc4221bf800, 0x3, 0xc424a16d40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424586000, 0x3c, 0x3c, 0x15179d56d6c46070, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc424586000, 0x3c, 0x3c, 0x15179d56d6c46070, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42377a3c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 114408 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42275a9c0, 0x15179d5711ad4378, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42275a9c0, 0x15179d5711ad4378, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42275a9c0, 0x15179d5711ad4378, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42275a930, 0x15179d5711ad4378, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42275a8d0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42275a8d0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42275a8d0, 0x5c4d11c, 0x10, 0xc4225bd9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42275a8d0, 0xc42371e438, 0xc42275a8d0, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42275a8d0, 0xc42371e438, 0x1f, 0xc42072de30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420f70690, 0xc42371e420, 0x6a551e0, 0xc4242642c0, 0xc400000008, 0xc4241ccec0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420f70690, 0x6a551e0, 0xc4242642c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420f70690, 0x6a551e0, 0xc4242642c0, 0xc42371e210, 0xc421a1c060) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 115283 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc423642f00, 0x45563f3, 0xc423bf91e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc423642f00, 0x5d2c558, 0xc423bf9420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42359dd10, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42359dd10, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79cf8, 0x5b6ec40, 0xc42395d2c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc424a5a600, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79cf8, 0x5b6ec40, 0xc42395d2c0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc424a5a600, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79cf8, 0x5b6ec40, 0xc42395d2c0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc424a5a600, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a79cf8, 0x5b6ec40, 0xc42395d2c0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc421ac8418, 0x6a551e0, 0xc424a5a600, 0xc422a79cf8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7c28e7b0c) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421081030, 0xc422a79ce0, 0x6a551e0, 0xc424a5a600, 0xc400000008, 0xc424a5a3c0, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421081030, 0x6a551e0, 0xc424a5a600, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421081030, 0x6a551e0, 0xc424a5a600, 0xc422a79c30, 0xc42395d1a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114902 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4202d8b40, 0x6a552a0, 0xc420bd42d0, 0xc4202d8b90, 0x7, 0xc4234e7a40, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4202d8b40, 0x6a552a0, 0xc420bd42d0, 0x7, 0xc4234e7a40, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4202d8b40, 0x6a552a0, 0xc420bd42d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4202d8b40, 0x6a552a0, 0xc420bd42d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4202d8b40, 0x6a552a0, 0xc420bd42d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00a00, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4202d8b40, 0x6a552a0, 0xc420bd42d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc421f00900, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc42117cc40, 0x6a552a0, 0xc42380a4b0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc42117cc40, 0x6a552a0, 0xc42380a4b0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc42373ffb0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc4204ba360, 0x6a552a0, 0xc42373ffb0, 0xc422769350, 0x29, 0x6a652e0, 0xc42048e3c0, 0xc42032fd80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 109816 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc422d99230, 0xc4206a4230, 0x7, 0xc420bd39a0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc422d99230, 0x7, 0xc420bd39a0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc422d99230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc422d99230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc422d99230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc422d99230, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc424838380, 0x6a552a0, 0xc422d99110, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc42314ef78, 0xc424dc1c00, 0x4, 0xc420c334c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc424dc1c00, 0x5, 0xc420c33480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).maybePushTransactions(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423764000, 0x34, 0x34, 0x15179d56ab784148, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:246 +0xb8e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc423764000, 0x34, 0x34, 0x15179d56ab784148, 0xc400000000, 0xc400000002, 0x10, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:339 +0x113 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).processIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:319 +0x7f github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc42334ce10) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115123 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc4216841e0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4216841e0, 0x15179d571ed33ee0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4216841e0, 0x15179d571ed33ee0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4216841e0, 0x15179d571ed33ee0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4216841b0, 0x15179d571ed33ee0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421684150, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421684150, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421684150, 0x5c4d11c, 0x10, 0xc424bbb9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421684150, 0xc422c75a38, 0xc421684150, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421684150, 0xc422c75a38, 0x1f, 0xc420895630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4212e7420, 0xc422c75a20, 0x6a551e0, 0xc42160e000, 0xc400000008, 0xc42391d180, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4212e7420, 0x6a551e0, 0xc42160e000, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4212e7420, 0x6a551e0, 0xc42160e000, 0xc422c75970, 0xc422b685a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114462 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc420518e10, 0x6a552a0, 0xc42359d6b0, 0xc420518e60, 0x7, 0xc424783540, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc420518e10, 0x6a552a0, 0xc42359d6b0, 0x7, 0xc424783540, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc420518e10, 0x6a552a0, 0xc42359d6b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc420518e10, 0x6a552a0, 0xc42359d6b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc420518e10, 0x6a552a0, 0xc42359d6b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc420518e10, 0x6a552a0, 0xc42359d6b0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc420721300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc422e6bdc0, 0x6a552a0, 0xc420c0fd70, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc422e6bdc0, 0x6a552a0, 0xc420c0fd70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc420c0f8c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc420f747e0, 0x6a552a0, 0xc420c0f8c0, 0xc422f3dda0, 0x29, 0x6a652e0, 0xc420358700, 0xc4221713a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114696 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc42282d400, 0x45563f3, 0xc4235ad1e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc42282d400, 0x5d2c558, 0xc4235ad420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42370f4a0, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42370f4a0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421632648, 0x5b6ec40, 0xc421c30ae0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42378d200, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421632648, 0x5b6ec40, 0xc421c30ae0, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc42378d200, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421632648, 0x5b6ec40, 0xc421c30ae0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42378d200, 0x5c6d3d9, 0x21, 0x5c13340, 0xc421632648, 0x5b6ec40, 0xc421c30ae0, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4241ca9f8, 0x6a551e0, 0xc42378d200, 0xc421632648, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x8003f5d2e) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420269570, 0xc421632630, 0x6a551e0, 0xc42378d200, 0xc400000008, 0xc42141df00, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420269570, 0x6a551e0, 0xc42378d200, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420269570, 0x6a551e0, 0xc42378d200, 0xc4216324d0, 0xc421c30a80) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114328 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc4242852c0, 0x45563f3, 0xc424cf31e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc4242852c0, 0x5d2c558, 0xc424cf3420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc420fae9f0, 0xc42000cd20, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc420fae9f0, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a78d28, 0x5b6ec40, 0xc420e0ff80, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc4241d3640, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a78d28, 0x5b6ec40, 0xc420e0ff80, 0xc422188780, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422188780, 0x6a551e0, 0xc4241d3640, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a78d28, 0x5b6ec40, 0xc420e0ff80, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc4241d3640, 0x5c6d3d9, 0x21, 0x5c13340, 0xc422a78d28, 0x5b6ec40, 0xc420e0ff80, 0xc422188780, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f8cd0, 0x6a551e0, 0xc4241d3640, 0xc422a78d28, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7bb2d4068) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc4239391f0, 0xc422a78d10, 0x6a551e0, 0xc4241d3640, 0xc400000008, 0xc421451200, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc4239391f0, 0x6a551e0, 0xc4241d3640, 0xc420412130, 0xf, 0xc420bd00f0, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc4239391f0, 0x6a551e0, 0xc4241d3640, 0xc422a78c60, 0xc420e0fe60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114818 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422f617c0, 0x45563f3, 0xc4247271e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422f617c0, 0x5d2c558, 0xc424727420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc42275bf50, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc42275bf50, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42276a7a8, 0x5b6ec40, 0xc423ad6e40, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42456f300, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42276a7a8, 0x5b6ec40, 0xc423ad6e40, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc42456f300, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42276a7a8, 0x5b6ec40, 0xc423ad6e40, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42456f300, 0x5c6d3d9, 0x21, 0x5c13340, 0xc42276a7a8, 0x5b6ec40, 0xc423ad6e40, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc4204f8360, 0x6a551e0, 0xc42456f300, 0xc42276a7a8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x4015826) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc421a68000, 0xc42276a790, 0x6a551e0, 0xc42456f300, 0xc400000008, 0xc42403b900, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc421a68000, 0x6a551e0, 0xc42456f300, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc421a68000, 0x6a551e0, 0xc42456f300, 0xc42276a2c0, 0xc4233412c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114444 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bd5c0, 0x15179d5711bc7230, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc4218bd5c0, 0x15179d5711bc7230, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc4218bd5c0, 0x15179d5711bc7230, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4211167b0, 0x15179d5711bc7230, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc421116750, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc421116750, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc421116750, 0x5c4d11c, 0x10, 0xc4233219b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc421116750, 0xc423f5f8d8, 0xc421116750, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc421116750, 0xc423f5f8d8, 0x1f, 0xc420bcb630, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422993960, 0xc423f5f8c0, 0x6a551e0, 0xc42335ac00, 0xc400000008, 0xc421452240, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422993960, 0x6a551e0, 0xc42335ac00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422993960, 0x6a551e0, 0xc42335ac00, 0xc423f5f810, 0xc423140d20) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114844 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423406390, 0x15179d5719910de0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423406390, 0x15179d5719910de0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423406390, 0x15179d5719910de0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc4224f3bc0, 0x15179d5719910de0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc4224f3b60, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc422b33200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc4224f3b60, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc4224f3b30, 0x5c4d11c, 0x10, 0xc4233f32c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc4224f3b30, 0xc4225a5f10, 0xc4224f3b30, 0xc42316d568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc4224f3b30, 0xc4225a5f10, 0xc420b53500, 0xc4204f8a88, 0xc4232a4400) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4224f3590, 0x5c13340, 0xc4225a5f10, 0x5c13340, 0xc4225a5f10, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4224f3590, 0x5c13340, 0xc4225a5f10, 0xc4203e5b20, 0xc4203e5b40, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4224f3590, 0x5c13340, 0xc4225a5f10, 0xc4203e5b20, 0xc4203e5b40, 0x6, 0xc42316d968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4224f3590, 0xc4225a5ea0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc422ba4c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423bad680, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423bad680, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc423bad680) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114709 [select, 1 minutes]: github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).waitOnHeader(0xc422fafa40, 0x45563f3, 0xc4247f91e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:258 +0x190 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport.(*Stream).Header(0xc422fafa40, 0x5d2c558, 0xc4247f9420, 0x6a61b40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/transport/transport.go:298 +0x2f github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.recvResponse(0x6a552a0, 0xc4229a1170, 0xc420371120, 0x0, 0x6a54aa0, 0x6cf0d10, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:50 +0xac github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.invoke(0x6a552a0, 0xc4229a1170, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4238732a8, 0x5b6ec40, 0xc420c61800, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:302 +0x9c5 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingClientInterceptor.func1(0x6a551e0, 0xc42403ae40, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4238732a8, 0x5b6ec40, 0xc420c61800, 0xc422447680, 0x5d2c528, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/client.go:44 +0xe27 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*ClientConn).Invoke(0xc422447680, 0x6a551e0, 0xc42403ae40, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4238732a8, 0x5b6ec40, 0xc420c61800, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:149 +0xdd github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.Invoke(0x6a551e0, 0xc42403ae40, 0x5c6d3d9, 0x21, 0x5c13340, 0xc4238732a8, 0x5b6ec40, 0xc420c61800, 0xc422447680, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/call.go:159 +0xc1 github.com/cockroachdb/cockroach/pkg/roachpb.(*internalClient).Batch(0xc423f810d0, 0x6a551e0, 0xc42403ae40, 0xc4238732a8, 0x0, 0x0, 0x0, 0x1, 0x40147f7, 0x7fc73d608) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3663 +0xd2 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420d09880, 0xc423873290, 0x6a551e0, 0xc42403ae40, 0xc400000008, 0xc4232a4680, 0x41278a8) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:254 +0x221 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420d09880, 0x6a551e0, 0xc42403ae40, 0xc42035f7d0, 0xf, 0xc420ebf590, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420d09880, 0x6a551e0, 0xc42403ae40, 0xc4238731e0, 0xc423c20180) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114529 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420c5a4e0, 0x15179d57138222b8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421314800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc420c5a4e0, 0x15179d57138222b8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421314800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc420c5a4e0, 0x15179d57138222b8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421314800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc420c5a4b0, 0x15179d57138222b8, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421314800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc420c5a450, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc421314800, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc420c5a450, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc420c5a450, 0x5c4d11c, 0x10, 0xc423e6b9b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc420c5a450, 0xc422f124e8, 0xc420c5a450, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc420c5a450, 0xc422f124e8, 0x1f, 0xc420a7ee30, 0x1) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc422528310, 0xc422f124d0, 0x6a551e0, 0xc422a5a8c0, 0xc400000008, 0xc4220a9240, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc422528310, 0x6a551e0, 0xc422a5a8c0, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc422528310, 0x6a551e0, 0xc422a5a8c0, 0xc422f12420, 0xc423cf9b60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114209 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421a94720, 0xc4206a4230, 0x7, 0xc42484f8b0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421a94720, 0x7, 0xc42484f8b0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421a94720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421a94720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421a94720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421a94720, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc422e07880, 0x6a552a0, 0xc421a946f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc421ac7d00, 0xc423de3800, 0x0, 0xc421ac7e38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc423de3800, 0x1, 0xc421a94660) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc42245d9e0, 0xc420b6ef00, 0x5, 0x5, 0x15179d570d9c00f8, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc423b9b1d0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 goroutine 115048 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421a3bd70, 0xc4206a4230, 0x7, 0xc4230805a0, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421a3bd70, 0x7, 0xc4230805a0, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421a3bd70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421a3bd70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421a3bd70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524300, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421a3bd70, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc423524200, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeat(0xc421c3a540, 0x6a552a0, 0xc424596030, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1113 +0x403 github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).heartbeatLoop(0xc421c3a540, 0x6a552a0, 0xc424596030) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:990 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).updateState.func1(0x6a552a0, 0xc420bd5b30) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:1308 +0x3e github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420bd5b30, 0xc4201e1b30, 0x29, 0x6a652e0, 0xc420b88320, 0xc42048ca60) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:266 +0xe6 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:261 +0x133 goroutine 114607 [select]: github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423905e90, 0x15179d57157b3f50, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2746 +0x696 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc423905e90, 0x15179d57157b3f50, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc423905e90, 0x15179d57157b3f50, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc423905e30, 0x15179d57157b3f50, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc423905dd0, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x3ff0000000000000, 0xc423acf400, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc423905dd0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc423905da0, 0x5c4d11c, 0x10, 0xc424e332c8, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc423905da0, 0xc423d9d110, 0xc423905da0, 0xc424be3568, 0x4012dd3) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc423905da0, 0xc423d9d110, 0xc420b53500, 0xc4241ca6f8, 0xc424dce480) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler.func1(0x6a552a0, 0xc4239181e0, 0x5c13340, 0xc423d9d110, 0x5c13340, 0xc423d9d110, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3693 +0x86 github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x6a552a0, 0xc4239181e0, 0x5c13340, 0xc423d9d110, 0xc420349a60, 0xc420349a80, 0x0, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:48 +0xa54 github.com/cockroachdb/cockroach/pkg/rpc.NewServerWithInterceptor.func1(0x6a552a0, 0xc4239181e0, 0x5c13340, 0xc423d9d110, 0xc420349a60, 0xc420349a80, 0x6, 0xc424be3968, 0x4014f58, 0x70) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/rpc/context.go:165 +0xb9 github.com/cockroachdb/cockroach/pkg/roachpb._Internal_Batch_Handler(0x5c01400, 0xc420b53500, 0x6a552a0, 0xc4239181e0, 0xc423d9d0a0, 0xc420eaab00, 0x0, 0x0, 0x5b392e0, 0xc420e35c40) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.pb.go:3695 +0x16d github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).processUnaryRPC(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc421e17e00, 0xc420ea8780, 0x6c7c700, 0x0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:921 +0x96e github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).handleStream(0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc421e17e00, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:1143 +0x1528 github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc4203a2a30, 0xc4205e8420, 0x6a60d60, 0xc4212da000, 0xc421e17e00) /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:638 +0x9f created by github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc.(*Server).serveStreams.func1 /Users/andrei/work/src/github.com/cockroachdb/cockroach/vendor/google.golang.org/grpc/server.go:636 +0xa1 goroutine 114936 [semacquire, 1 minutes]: sync.runtime_SemacquireMutex(0xc4224e6794, 0xc422379700) /Users/andrei/work/src/go/src/runtime/sema.go:71 +0x3d sync.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/go/src/sync/mutex.go:134 +0xee github.com/cockroachdb/cockroach/pkg/util/syncutil.(*Mutex).Lock(0xc4224e6790) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/syncutil/mutex_sync.go:32 +0x2d github.com/cockroachdb/cockroach/pkg/storage.(*Replica).propose(0xc4224e6700, 0x6a552a0, 0xc42387bce0, 0x15179d5121f317b8, 0x0, 0x0, 0x200000002, 0x3, 0x0, 0xc4232235f0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:3067 +0xa99 github.com/cockroachdb/cockroach/pkg/storage.(*Replica).tryExecuteWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387bce0, 0x15179d571aceece0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2720 +0x3eb github.com/cockroachdb/cockroach/pkg/storage.(*Replica).executeWriteBatch(0xc4224e6700, 0x6a552a0, 0xc42387bce0, 0x15179d571aceece0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:2570 +0xae github.com/cockroachdb/cockroach/pkg/storage.(*Replica).Send(0xc4224e6700, 0x6a552a0, 0xc42387bce0, 0x15179d571aceece0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/replica.go:1873 +0x6a5 github.com/cockroachdb/cockroach/pkg/storage.(*Store).Send(0xc4206d1c00, 0x6a552a0, 0xc42387bc80, 0x15179d571aceece0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/store.go:2726 +0x65a github.com/cockroachdb/cockroach/pkg/storage.(*Stores).Send(0xc4210c44d0, 0x6a552a0, 0xc42387bc20, 0x0, 0x0, 0x200000002, 0x3, 0x7, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/stores.go:214 +0x214 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal.func1(0x6a552a0, 0xc42387bc20, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:897 +0x192 github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunTaskWithErr(0xc421163e60, 0x6a552a0, 0xc42387bc20, 0x5c4d11c, 0x10, 0xc4237839b0, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:245 +0xf7 github.com/cockroachdb/cockroach/pkg/server.(*Node).batchInternal(0xc420b53500, 0x6a552a0, 0xc42387bc20, 0xc423ba8648, 0xc42387bc20, 0x4aec0fe, 0x6a551e0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:884 +0x233 github.com/cockroachdb/cockroach/pkg/server.(*Node).Batch(0xc420b53500, 0x6a552a0, 0xc42387bc20, 0xc423ba8648, 0x1f, 0x0, 0x0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/server/node.go:925 +0x99 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send.func1(0xc420b61dc0, 0xc423ba8630, 0x6a551e0, 0xc424a16c00, 0xc400000008, 0xc4214539c0, 0x299a73a0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:243 +0x6f6 github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).send(0xc420b61dc0, 0x6a551e0, 0xc424a16c00, 0xc420375460, 0xf, 0xc422ff6d70, 0x0, 0x0, 0x200000002, 0x3, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:274 +0x9a github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext.func1(0xc420b61dc0, 0x6a551e0, 0xc424a16c00, 0xc4237dae70, 0xc4211c03c0) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:225 +0xb9 created by github.com/cockroachdb/cockroach/pkg/kv.(*grpcTransport).SendNext /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/transport.go:223 +0x1e1 goroutine 114537 [select]: github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendToReplicas(0xc4206a41e0, 0x6a552a0, 0xc421863a10, 0xc4206a4230, 0x7, 0xc420b96d70, 0x3, 0x3, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1302 +0x506 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendRPC(0xc4206a41e0, 0x6a552a0, 0xc421863a10, 0x7, 0xc420b96d70, 0x3, 0x3, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:396 +0x27b github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendSingleRange(0xc4206a41e0, 0x6a552a0, 0xc421863a10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:463 +0x21a github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).sendPartialBatch(0xc4206a41e0, 0x6a552a0, 0xc421863a10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1071 +0x34f github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc4206a41e0, 0x6a552a0, 0xc421863a10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:742 +0x16c8 github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc4206a41e0, 0x6a552a0, 0xc421863a10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:658 +0x41f github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xc421a81880, 0x6a552a0, 0xc4218639e0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:490 +0x1f3 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:553 +0x118 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).send(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:535 +0xa5 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).(github.com/cockroachdb/cockroach/pkg/internal/client.send)-fm(0x6a552a0, 0xc420e48f90, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x76 github.com/cockroachdb/cockroach/pkg/internal/client.sendAndFill(0x6a552a0, 0xc420e48f90, 0xc423e1fd00, 0xc422169c00, 0x0, 0xc423e1fe38) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:468 +0xf8 github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).Run(0xc4211dbef0, 0x6a552a0, 0xc420e48f90, 0xc422169c00, 0x1, 0xc421863980) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:491 +0x94 github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupFinishedTxnIntents(0xc421013680, 0x6a552a0, 0xc420e48f90, 0xc421b64360, 0xc4211def00, 0x5, 0x5, 0x15179d5713c69628, 0xc400000000, 0x402dd49, ...) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:536 +0x46e github.com/cockroachdb/cockroach/pkg/storage.(*intentResolver).cleanupTxnIntentsAsync.func1(0x6a552a0, 0xc420e48f90) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/storage/intent_resolver.go:387 +0x11d github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask.func1(0xc421163e60, 0x6a552a0, 0xc420e48f90, 0x5c85a6a, 0x2a, 0xc42126e6c0, 0x0, 0x0, 0xc421862720) /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:327 +0x110 created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunLimitedAsyncTask /Users/andrei/work/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:321 +0x296 exit status 2 FAIL github.com/cockroachdb/cockroach/pkg/sql 120.846s