panic: pebble: closed [recovered]
        panic: pebble: closed [recovered]
        panic: pebble: closed

goroutine 9311 [running]:
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).Recover(0xc001803b00, 0x1702240, 0xc000431680)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:183 +0x11f
panic(0x9b3820, 0xc0002931a0)
        /usr/local/go/src/runtime/panic.go:969 +0x166
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send.func1(0xc000b1af58, 0xc000b1afe0, 0xc000b1afd8, 0xc000476e00)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:103 +0x1e6
panic(0x9b3820, 0xc0002931a0)
        /usr/local/go/src/runtime/panic.go:975 +0x3e3
github.com/cockroachdb/pebble.(*DB).newIterInternal(0xc000441500, 0x177b5a0, 0xc000bf8280, 0x0, 0x0, 0x0, 0xc00095c370, 0x0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/db.go:668 +0xd94
github.com/cockroachdb/pebble.(*Batch).NewIter(0xc00043b180, 0xc00095c370, 0x0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/vendor/github.com/cockroachdb/pebble/batch.go:675 +0x1df
github.com/cockroachdb/cockroach/pkg/storage.(*pebbleIterator).init(0xc00095c368, 0x16e6680, 0xc00043b180, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/pebble_iterator.go:125 +0x46b
github.com/cockroachdb/cockroach/pkg/storage.(*pebbleBatch).NewIterator(0xc00095c340, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/pebble_batch.go:207 +0x14f
github.com/cockroachdb/cockroach/pkg/storage.MVCCGet(0x1702300, 0xc001d389c0, 0x82ece9408, 0xc001204780, 0xc0004461c0, 0x26, 0x40, 0x0, 0x0, 0xc000000000, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:754 +0x9b
github.com/cockroachdb/cockroach/pkg/storage.MVCCGetProto(0x1702300, 0xc001d389c0, 0x82ece9408, 0xc001204780, 0xc0004461c0, 0x26, 0x40, 0x0, 0x0, 0x173e060, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/storage/mvcc.go:630 +0xd7
github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval.EndTxn(0x1702300, 0xc001d389c0, 0x82ed61598, 0xc001204780, 0x179c260, 0xc00038b000, 0x7b, 0x3, 0x100000001, 0x1, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval/cmd_end_transaction.go:200 +0x309
github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateCommand(0x1702300, 0xc001d389c0, 0xc00058a190, 0x8, 0x0, 0x82ed61598, 0xc001204780, 0x179c260, 0xc00038b000, 0xc000734000, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_evaluate.go:471 +0x235
github.com/cockroachdb/cockroach/pkg/kv/kvserver.evaluateBatch(0x1702300, 0xc001d389c0, 0xc00058a190, 0x8, 0x82ed61598, 0xc001204780, 0x179c260, 0xc00038b000, 0xc000734000, 0xc000bf8080, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_evaluate.go:241 +0x3c2
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWrapper(0xc00038b000, 0x1702300, 0xc001d389c0, 0xc00058a190, 0x8, 0x179c260, 0xc00038b000, 0xc000734000, 0xc000bf8080, 0xc0012043c0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_write.go:557 +0x144
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatchWithServersideRefreshes(0xc00038b000, 0x1702300, 0xc001d389c0, 0xc00058a190, 0x8, 0x179c260, 0xc00038b000, 0xc000734000, 0xc000bf8080, 0xc0012043c0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_write.go:526 +0x135
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateWriteBatch(0xc00038b000, 0x1702300, 0xc001d389c0, 0xc00058a190, 0x8, 0xc000bf8080, 0xc0012043c0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_write.go:349 +0x212
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evaluateProposal(0xc00038b000, 0x1702300, 0xc001d389c0, 0xc00058a190, 0x8, 0xc000bf8080, 0xc0012043c0, 0x0, 0xc000aa5600, 0x0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_proposal.go:735 +0x127
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).requestToProposal(0xc00038b000, 0x1702300, 0xc001d389c0, 0xc00058a190, 0x8, 0xc000bf8080, 0xc0012043c0, 0x0, 0x0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_proposal.go:855 +0x8e
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).evalAndPropose(0xc00038b000, 0x1702300, 0xc001d389c0, 0xc000bf8080, 0xc000744000, 0xc000b1a330, 0x0, 0x0, 0x0, 0x0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_raft.go:73 +0xee
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeWriteBatch(0xc00038b000, 0x1702300, 0xc001d389c0, 0xc000bf8080, 0x0, 0x0, 0xc0004b2da0, 0x100000001, 0x1, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_write.go:133 +0x769
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).executeBatchWithConcurrencyRetries(0xc00038b000, 0x1702300, 0xc001d389c0, 0xc000bf8080, 0xdfed58, 0x0, 0x0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:275 +0x3e7
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).sendWithRangeID(0xc00038b000, 0x1702300, 0xc001d38990, 0x2, 0xc000bf8080, 0x0, 0x0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:95 +0x6b2
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Replica).Send(0xc00038b000, 0x1702300, 0xc001d38990, 0x7b, 0x3, 0x100000001, 0x1, 0x0, 0x2, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/replica_send.go:37 +0x91
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Store).Send(0xc000476e00, 0x1702300, 0xc001d38810, 0x7b, 0x3, 0x100000001, 0x1, 0x0, 0x2, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/store_send.go:194 +0x5a2
github.com/cockroachdb/cockroach/pkg/kv/kvserver.(*Stores).Send(0xc000caf300, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x2, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvserver/stores.go:177 +0xed
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*senderTransport).SendNext(0xc000b04000, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x100000001, 0x1, 0x0, 0x2, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/transport.go:299 +0x21e
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*localTestClusterTransport).SendNext(0xc000f200a0, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/local_test_cluster_util.go:44 +0x8f
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendToReplicas(0xc000578840, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1795 +0x6b1
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).sendPartialBatch(0xc000578840, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1459 +0x305
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).divideAndSendBatchToRanges(0xc000578840, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:1099 +0x18ef
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*DistSender).Send(0xc000578840, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/dist_sender.go:739 +0x8e4
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnLockGatekeeper).SendLocked(0xc00014cd38, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_lock_gatekeeper.go:86 +0x11c
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnMetricRecorder).SendLocked(0xc00014cd00, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_metric_recorder.go:46 +0x8d
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnCommitter).SendLocked(0xc00014ccd0, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_committer.go:190 +0x53b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xc00014cc38, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:247 +0x9b
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSpanRefresher).SendLocked(0xc00014cc38, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_span_refresher.go:182 +0x180
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnPipeliner).SendLocked(0xc00014cb78, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_pipeliner.go:252 +0x159
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnSeqNumAllocator).SendLocked(0xc00014cb58, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_seq_num_allocator.go:105 +0x20c
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*txnHeartbeater).SendLocked(0xc00014cab8, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_interceptor_heartbeater.go:172 +0x1a9
github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord.(*TxnCoordSender).Send(0xc00014c900, 0x1702300, 0xc001d38810, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord/txn_coord_sender.go:499 +0x3cc
github.com/cockroachdb/cockroach/pkg/kv.(*DB).sendUsingSender(0xc000caf400, 0x17022c0, 0xc001204360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/db.go:742 +0x122
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).Send(0xc0016e6360, 0x17022c0, 0xc001204360, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:911 +0x11e
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).rollback.func1.1(0x17022c0, 0xc001204360, 0xb2d05e00, 0x17022c0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:733 +0x98
github.com/cockroachdb/cockroach/pkg/util/contextutil.RunWithTimeout(0x17022c0, 0xc001204360, 0xc8b302, 0x12, 0xb2d05e00, 0xc000bcee68, 0x0, 0x0)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/util/contextutil/context.go:135 +0x9e
github.com/cockroachdb/cockroach/pkg/kv.(*Txn).rollback.func1(0x1702240, 0xc000431680)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn.go:732 +0x14b
github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask.func1(0xc001803b00, 0x1702240, 0xc000431680, 0xc000d34020, 0x16, 0x0, 0x0, 0xc001d78140)
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:323 +0xee
created by github.com/cockroachdb/cockroach/pkg/util/stop.(*Stopper).RunAsyncTask
        /data/home/kena/src/go/src/github.com/cockroachdb/cockroach/pkg/util/stop/stopper.go:318 +0x131

make stress PKG=./pkg/kv/kvclient/kvcoord