# Don't forget to check out a clean suitable branch and experiment with the
# stress invocation until the desired results present themselves. For example,
# using stress instead of stressrace and passing the '-p' stressflag which
# controls concurrency.
./scripts/gceworker.sh start && ./scripts/gceworker.sh mosh
cd ~/go/src/github.com/cockroachdb/cockroach && \
stdbuf -oL -eL \
make stressrace TESTS=scaledata/distributed_semaphore/nodes=3 PKG=roachtest TESTTIMEOUT=5m STRESSFLAGS='-maxtime 20m -timeout 10m' 2>&1 | tee /tmp/stress.log

The test failed on branch=provisional_201909060000_v19.2.0-beta.20190910, cloud=gce:
test artifacts and logs in: /home/agent/work/.go/src/github.com/cockroachdb/cockroach/artifacts/20190906-1472753/scaledata/distributed_semaphore/nodes=3/run_1
	cluster.go:2114,scaledata.go:121,scaledata.go:48,test_runner.go:688: /home/agent/work/.go/src/github.com/cockroachdb/cockroach/bin/roachprod run teamcity-1567786968-05-n4cpu4:4 -- ./distributed_semaphore  --duration_secs=600 --num_workers=16 --cockroach_ip_addresses_csv='10.128.0.57:26257,10.128.0.65:26257,10.128.0.22:26257'  returned:
		stderr:
		Error while dialing dial tcp 10.128.0.22:26257: connect: connection refused": ... Retrying after sleeping 10ns
		2019/09/06 16:28:39 RobustDB.RandomDB chose DB at index 0
		2019/09/06 16:28:39 [/Users/nathan/Go/src/github.com/scaledata/rksql/src/go/src/rubrik/sqlapp/tx.go:43 dial tcp 10.128.0.22:26257: connect: connection refused]
		2019/09/06 16:28:39 ExecuteTx retry attempt 3 failed, started at 2019-09-06 16:28:39.807905278 +0000 UTC m=+88.525719942, now = 2019-09-06 16:28:39.808539856 +0000 UTC m=+88.526354546, took 634.604µs
		2019/09/06 16:28:39 [/Users/nathan/Go/src/github.com/scaledata/rksql/src/go/src/rubrik/sqlapp/tx.go:43 dial tcp 10.128.0.22:26257: connect: connection refused]
		2019/09/06 16:28:39 RobustDB.RandomDB chose DB at index 0
		2019/09/06 16:28:39 Attempt failed with error dial tcp 10.128.0.22:26257: connect: connection refused: ... Retrying after sleeping 20ns
		2019/09/06 16:28:39 RobustDB.RandomDB chose DB at index 0
		2019/09/06 16:28:39 RobustDB.RandomDB chose DB at index 0
		Error:  exit status 255
		
		stdout:
		pq: restart transaction: TransactionRetryWithProtoRefreshError: WriteTooOldError: write at timestamp 1567787231.290698393,0 too old; wrote at 1567787231.313988234,11
		pq: restart transaction: TransactionRetryWithProtoRefreshError: ReadWithinUncertaintyIntervalError: read at time 1567787231.296866340,0 encountered previous write with future timestamp 1567787231.299592850,54 within uncertainty interval `t <= 1567787231.313988234,4`; observed timestamps: [{1 1567787231.313988234,4} {2 1567787231.296866340,0}]
		pq: restart transaction: TransactionRetryWithProtoRefreshError: ReadWithinUncertaintyIntervalError: read at time 1567787231.293855505,0 encountered previous write with future timestamp 1567787231.299592850,54 within uncertainty interval `t <= 1567787231.313984783,6`; observed timestamps: [{1 1567787231.313984783,6} {2 1567787231.293855505,0}]
		: exit status 1