who_has not set for task in state fetch #5751
Description
Known tests to be affected
In production environment validation is disabled which will likely cause other errors and/or deadlocks.
Traceback (most recent call last):
File "/home/runner/work/distributed/distributed/distributed/worker.py", line 3067, in gather_dep
response = await get_data_from_worker(
File "/home/runner/work/distributed/distributed/distributed/worker.py", line 4353, in get_data_from_worker
return await retry_operation(_get_data, operation="get_data_from_worker")
File "/home/runner/work/distributed/distributed/distributed/utils_comm.py", line 385, in retry_operation
return await retry(
File "/home/runner/work/distributed/distributed/distributed/utils_comm.py", line 370, in retry
return await coro()
File "/home/runner/work/distributed/distributed/distributed/worker.py", line 4330, in _get_data
comm = await rpc.connect(worker)
File "/home/runner/work/distributed/distributed/distributed/core.py", line 1067, in connect
comm = await fut
File "/home/runner/work/distributed/distributed/distributed/comm/core.py", line 329, in connect
raise OSError(
OSError: Timed out during handshake while connecting to tcp://127.0.0.1:36081 after 30 s
distributed.worker - ERROR -
Traceback (most recent call last):
File "/home/runner/work/distributed/distributed/distributed/worker.py", line 3986, in validate_task
self.validate_task_fetch(ts)
File "/home/runner/work/distributed/distributed/distributed/worker.py", line 3928, in validate_task_fetch
assert ts.who_has
Task story
(
"inc-aa226ae3a9f799819e1e685fba467442",
"compute-task",
"compute-task-1643835890.882272",
1643835890.8875856,
),
(
"inc-aa226ae3a9f799819e1e685fba467442",
"released",
"waiting",
"waiting",
{"inc-aa226ae3a9f799819e1e685fba467442": "ready"},
"compute-task-1643835890.882272",
1643835890.887607,
),
(
"inc-aa226ae3a9f799819e1e685fba467442",
"waiting",
"ready",
"ready",
{},
"compute-task-1643835890.882272",
1643835890.8876154,
),
(
"inc-aa226ae3a9f799819e1e685fba467442",
"ensure-task-exists",
"ready",
"compute-task-1643835890.8866765",
1643835890.8877435,
),
(
"inc-aa226ae3a9f799819e1e685fba467442",
"release-key",
"compute-task-1643835890.8866765",
1643835890.8879642,
),
(
"inc-aa226ae3a9f799819e1e685fba467442",
"ready",
"released",
"released",
{},
"compute-task-1643835890.8866765",
1643835890.8879726,
),
(
"inc-aa226ae3a9f799819e1e685fba467442",
"released",
"fetch",
"fetch",
{},
"compute-task-1643835890.8866765",
1643835890.8879821,
),
(
"inc-aa226ae3a9f799819e1e685fba467442",
"ready",
"fetch",
"fetch",
{},
"compute-task-1643835890.8866765",
1643835890.8879848,
),
Full story
AssertionError: Invalid TaskState encountered for <TaskState 'inc-aa226ae3a9f799819e1e685fba467442' fetch>.
Story:
[('inc-aa226ae3a9f799819e1e685fba467442', 'ensure-task-exists', 'released', 'compute-task-1643835890.875481', 1643835890.8762364), ('inc-aa226ae3a9f799819e1e685fba467442', 'released', 'fetch', 'fetch', {}, 'compute-task-1643835890.875481', 1643835890.876392), ('gather-dependencies', 'tcp://127.0.0.1:36081', {'inc-0c3f7739638b70e403ab21f80b09970b', 'inc-74fa2a3ce29de41facb1b01e75154df2', 'inc-bc330a1ea35cf0e28ef6270b770981cb', 'inc-aa226ae3a9f799819e1e685fba467442', 'inc-1bc6cda651ec56203f196053394876a1'}, 'ensure-communicating-1643835890.8765168', 1643835890.8765872), ('inc-aa226ae3a9f799819e1e685fba467442', 'fetch', 'flight', 'flight', {}, 'ensure-communicating-1643835890.8765168', 1643835890.876613), ('request-dep', 'tcp://127.0.0.1:36081', {'inc-0c3f7739638b70e403ab21f80b09970b', 'inc-74fa2a3ce29de41facb1b01e75154df2', 'inc-bc330a1ea35cf0e28ef6270b770981cb', 'inc-aa226ae3a9f799819e1e685fba467442', 'inc-1bc6cda651ec56203f196053394876a1'}, 'ensure-communicating-1643835890.8765168', 1643835890.876833), ('receive-dep-failed', 'tcp://127.0.0.1:36081', {'inc-0c3f7739638b70e403ab21f80b09970b', 'inc-74fa2a3ce29de41facb1b01e75154df2', 'inc-bc330a1ea35cf0e28ef6270b770981cb', 'inc-aa226ae3a9f799819e1e685fba467442', 'inc-1bc6cda651ec56203f196053394876a1'}, 'ensure-communicating-1643835890.8765168', 1643835890.8813467), ('inc-aa226ae3a9f799819e1e685fba467442', 'missing-dep', 'ensure-communicating-1643835890.8765168', 1643835890.8813949), ('inc-aa226ae3a9f799819e1e685fba467442', 'flight', 'missing', 'missing', {}, 'ensure-communicating-1643835890.8765168', 1643835890.881428), ('inc-aa226ae3a9f799819e1e685fba467442', 'release-key', 'processing-released-1643835890.880434', 1643835890.8822312), ('inc-aa226ae3a9f799819e1e685fba467442', 'missing', 'released', 'released', {'inc-aa226ae3a9f799819e1e685fba467442': 'forgotten'}, 'processing-released-1643835890.880434', 1643835890.8822412), ('inc-aa226ae3a9f799819e1e685fba467442', 'released', 'forgotten', 'forgotten', {}, 'processing-released-1643835890.880434', 1643835890.882249), ('inc-aa226ae3a9f799819e1e685fba467442', 'compute-task', 'compute-task-1643835890.880547', 1643835890.8824306), ('inc-aa226ae3a9f799819e1e685fba467442', 'released', 'waiting', 'waiting', {'inc-aa226ae3a9f799819e1e685fba467442': 'ready'}, 'compute-task-1643835890.880547', 1643835890.882451), ('inc-aa226ae3a9f799819e1e685fba467442', 'waiting', 'ready', 'ready', {}, 'compute-task-1643835890.880547', 1643835890.8824592), ('inc-aa226ae3a9f799819e1e685fba467442', 'ready', 'executing', 'executing', {}, 'ensure-computing-1643835890.8831396', 1643835890.8831706), ('inc-aa226ae3a9f799819e1e685fba467442', 'put-in-memory', 'ensure-computing-1643835890.8831396', 1643835890.8835356), ('inc-aa226ae3a9f799819e1e685fba467442', 'executing', 'memory', 'memory', {}, 'ensure-computing-1643835890.8831396', 1643835890.883553), ('free-keys', ('inc-aa226ae3a9f799819e1e685fba467442',), 'processing-released-1643835890.8822534', 1643835890.8875349), ('inc-aa226ae3a9f799819e1e685fba467442', 'release-key', 'processing-released-1643835890.8822534', 1643835890.8875427), ('inc-aa226ae3a9f799819e1e685fba467442', 'memory', 'released', 'released', {'inc-aa226ae3a9f799819e1e685fba467442': 'forgotten'}, 'processing-released-1643835890.8822534', 1643835890.8875651), ('inc-aa226ae3a9f799819e1e685fba467442', 'released', 'forgotten', 'forgotten', {}, 'processing-released-1643835890.8822534', 1643835890.8875735), ('inc-aa226ae3a9f799819e1e685fba467442', 'compute-task', 'compute-task-1643835890.882272', 1643835890.8875856), ('inc-aa226ae3a9f799819e1e685fba467442', 'released', 'waiting', 'waiting', {'inc-aa226ae3a9f799819e1e685fba467442': 'ready'}, 'compute-task-1643835890.882272', 1643835890.887607), ('inc-aa226ae3a9f799819e1e685fba467442', 'waiting', 'ready', 'ready', {}, 'compute-task-1643835890.882272', 1643835890.8876154), ('inc-aa226ae3a9f799819e1e685fba467442', 'ensure-task-exists', 'ready', 'compute-task-1643835890.8866765', 1643835890.8877435), ('inc-aa226ae3a9f799819e1e685fba467442', 'release-key', 'compute-task-1643835890.8866765', 1643835890.8879642), ('inc-aa226ae3a9f799819e1e685fba467442', 'ready', 'released', 'released', {}, 'compute-task-1643835890.8866765', 1643835890.8879726), ('inc-aa226ae3a9f799819e1e685fba467442', 'released', 'fetch', 'fetch', {}, 'compute-task-1643835890.8866765', 1643835890.8879821), ('inc-aa226ae3a9f799819e1e685fba467442', 'ready', 'fetch', 'fetch', {}, 'compute-task-1643835890.8866765', 1643835890.8879848)]