-
-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Closed
Labels
Description
cc @convexset any thoughts on how to avoid this intermittent failure in the tsqr tests?
In [1]: from dask.array.tests.test_linalg import test_tsqr_uncertain
In [2]: for i in range(1000): test_tsqr_uncertain(40, 5, (10, 5), True, True, None)
---------------------------------------------------------------------------
LinAlgError Traceback (most recent call last)
<ipython-input-2-8e08b9f31742> in <module>()
----> 1 for i in range(1000): test_tsqr_uncertain(40, 5, (10, 5), True, True, None)
~/workspace/dask/dask/array/tests/test_linalg.py in test_tsqr_uncertain(m_min, n_max, chunks, vary_rows, vary_cols, error_type)
131 # test QR
132 q, r = tsqr(data)
--> 133 q = q.compute() # because uncertainty
134 r = r.compute()
135 assert_eq((m_q, n_q), q.shape) # shape check
~/workspace/dask/dask/base.py in compute(self, **kwargs)
154 dask.base.compute
155 """
--> 156 (result,) = compute(self, traverse=False, **kwargs)
157 return result
158
~/workspace/dask/dask/base.py in compute(*args, **kwargs)
400 keys = [x.__dask_keys__() for x in collections]
401 postcomputes = [x.__dask_postcompute__() for x in collections]
--> 402 results = schedule(dsk, keys, **kwargs)
403 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
404
~/workspace/dask/dask/local.py in get_sync(dsk, keys, **kwargs)
560 """
561 kwargs.pop('num_workers', None) # if num_workers present, remove it
--> 562 return get_async(apply_sync, 1, dsk, keys, **kwargs)
563
564
~/workspace/dask/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
527
528 while state['ready'] and len(state['running']) < num_workers:
--> 529 fire_task()
530
531 succeeded = True
~/workspace/dask/dask/local.py in fire_task()
502 args=(key, dumps((dsk[key], data)),
503 dumps, loads, get_id, pack_exception),
--> 504 callback=queue.put)
505
506 # Seed initial tasks into the thread pool
~/workspace/dask/dask/local.py in apply_sync(func, args, kwds, callback)
549 def apply_sync(func, args=(), kwds={}, callback=None):
550 """ A naive synchronous version of apply_async """
--> 551 res = func(*args, **kwds)
552 if callback is not None:
553 callback(res)
~/workspace/dask/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
293 failed = False
294 except BaseException as e:
--> 295 result = pack_exception(e, dumps)
296 failed = True
297 return key, result, failed
~/workspace/dask/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
288 try:
289 task, data = loads(task_info)
--> 290 result = _execute_task(task, data)
291 id = get_id()
292 result = dumps((result, id))
~/workspace/dask/dask/local.py in _execute_task(arg, cache, dsk)
269 func, args = arg[0], arg[1:]
270 args2 = [_execute_task(a, cache) for a in args]
--> 271 return func(*args2)
272 elif not ishashable(arg):
273 return arg
~/Software/anaconda/lib/python3.6/site-packages/numpy/linalg/linalg.py in qr(a, mode)
752 a, wrap = _makearray(a)
753 _assertRank2(a)
--> 754 _assertNoEmpty2d(a)
755 m, n = a.shape
756 t, result_t = _commonType(a)
~/Software/anaconda/lib/python3.6/site-packages/numpy/linalg/linalg.py in _assertNoEmpty2d(*arrays)
223 for a in arrays:
224 if _isEmpty2d(a):
--> 225 raise LinAlgError("Arrays cannot be empty")
226
227 def transpose(a):
LinAlgError: Arrays cannot be emptyIn [3]: debug
> /home/mrocklin/Software/anaconda/lib/python3.6/site-packages/numpy/linalg/linalg.py(225)_assertNoEmpty2d()
223 for a in arrays:
224 if _isEmpty2d(a):
--> 225 raise LinAlgError("Arrays cannot be empty")
226
227 def transpose(a):
ipdb> pp a
array([], shape=(0, 3), dtype=float64)
ipdb> up
> /home/mrocklin/Software/anaconda/lib/python3.6/site-packages/numpy/linalg/linalg.py(754)qr()
752 a, wrap = _makearray(a)
753 _assertRank2(a)
--> 754 _assertNoEmpty2d(a)
755 m, n = a.shape
756 t, result_t = _commonType(a)
ipdb>
> /home/mrocklin/workspace/dask/dask/local.py(271)_execute_task()
269 func, args = arg[0], arg[1:]
270 args2 = [_execute_task(a, cache) for a in args]
--> 271 return func(*args2)
272 elif not ishashable(arg):
273 return arg
ipdb> pp func
<function qr at 0x7feba447a9d8>
ipdb> pp args2
[array([], shape=(0, 3), dtype=float64)]
ipdb> up
> /home/mrocklin/workspace/dask/dask/local.py(296)execute_task()
294 except BaseException as e:
295 result = pack_exception(e, dumps)
--> 296 failed = True
297 return key, result, failed
298
ipdb> pp key
('qr-bb4232513fb6d6a0b66ccbea15fe7885', 5, 0)Reactions are currently unavailable