Skip to content

Intermittent failure for test_tsqr_uncertain #3659

@mrocklin

Description

@mrocklin

cc @convexset any thoughts on how to avoid this intermittent failure in the tsqr tests?

In [1]: from dask.array.tests.test_linalg import test_tsqr_uncertain

In [2]: for i in range(1000): test_tsqr_uncertain(40, 5, (10, 5), True, True, None)
---------------------------------------------------------------------------
LinAlgError                               Traceback (most recent call last)
<ipython-input-2-8e08b9f31742> in <module>()
----> 1 for i in range(1000): test_tsqr_uncertain(40, 5, (10, 5), True, True, None)

~/workspace/dask/dask/array/tests/test_linalg.py in test_tsqr_uncertain(m_min, n_max, chunks, vary_rows, vary_cols, error_type)
    131             # test QR
    132             q, r = tsqr(data)
--> 133             q = q.compute()  # because uncertainty
    134             r = r.compute()
    135             assert_eq((m_q, n_q), q.shape)  # shape check

~/workspace/dask/dask/base.py in compute(self, **kwargs)
    154         dask.base.compute
    155         """
--> 156         (result,) = compute(self, traverse=False, **kwargs)
    157         return result
    158 

~/workspace/dask/dask/base.py in compute(*args, **kwargs)
    400     keys = [x.__dask_keys__() for x in collections]
    401     postcomputes = [x.__dask_postcompute__() for x in collections]
--> 402     results = schedule(dsk, keys, **kwargs)
    403     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    404 

~/workspace/dask/dask/local.py in get_sync(dsk, keys, **kwargs)
    560     """
    561     kwargs.pop('num_workers', None)    # if num_workers present, remove it
--> 562     return get_async(apply_sync, 1, dsk, keys, **kwargs)
    563 
    564 

~/workspace/dask/dask/local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
    527 
    528                 while state['ready'] and len(state['running']) < num_workers:
--> 529                     fire_task()
    530 
    531             succeeded = True

~/workspace/dask/dask/local.py in fire_task()
    502                             args=(key, dumps((dsk[key], data)),
    503                                   dumps, loads, get_id, pack_exception),
--> 504                             callback=queue.put)
    505 
    506             # Seed initial tasks into the thread pool

~/workspace/dask/dask/local.py in apply_sync(func, args, kwds, callback)
    549 def apply_sync(func, args=(), kwds={}, callback=None):
    550     """ A naive synchronous version of apply_async """
--> 551     res = func(*args, **kwds)
    552     if callback is not None:
    553         callback(res)

~/workspace/dask/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    293         failed = False
    294     except BaseException as e:
--> 295         result = pack_exception(e, dumps)
    296         failed = True
    297     return key, result, failed

~/workspace/dask/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    288     try:
    289         task, data = loads(task_info)
--> 290         result = _execute_task(task, data)
    291         id = get_id()
    292         result = dumps((result, id))

~/workspace/dask/dask/local.py in _execute_task(arg, cache, dsk)
    269         func, args = arg[0], arg[1:]
    270         args2 = [_execute_task(a, cache) for a in args]
--> 271         return func(*args2)
    272     elif not ishashable(arg):
    273         return arg

~/Software/anaconda/lib/python3.6/site-packages/numpy/linalg/linalg.py in qr(a, mode)
    752     a, wrap = _makearray(a)
    753     _assertRank2(a)
--> 754     _assertNoEmpty2d(a)
    755     m, n = a.shape
    756     t, result_t = _commonType(a)

~/Software/anaconda/lib/python3.6/site-packages/numpy/linalg/linalg.py in _assertNoEmpty2d(*arrays)
    223     for a in arrays:
    224         if _isEmpty2d(a):
--> 225             raise LinAlgError("Arrays cannot be empty")
    226 
    227 def transpose(a):

LinAlgError: Arrays cannot be empty
In [3]: debug
> /home/mrocklin/Software/anaconda/lib/python3.6/site-packages/numpy/linalg/linalg.py(225)_assertNoEmpty2d()
    223     for a in arrays:
    224         if _isEmpty2d(a):
--> 225             raise LinAlgError("Arrays cannot be empty")
    226 
    227 def transpose(a):

ipdb> pp a
array([], shape=(0, 3), dtype=float64)
ipdb> up
> /home/mrocklin/Software/anaconda/lib/python3.6/site-packages/numpy/linalg/linalg.py(754)qr()
    752     a, wrap = _makearray(a)
    753     _assertRank2(a)
--> 754     _assertNoEmpty2d(a)
    755     m, n = a.shape
    756     t, result_t = _commonType(a)

ipdb> 
> /home/mrocklin/workspace/dask/dask/local.py(271)_execute_task()
    269         func, args = arg[0], arg[1:]
    270         args2 = [_execute_task(a, cache) for a in args]
--> 271         return func(*args2)
    272     elif not ishashable(arg):
    273         return arg

ipdb> pp func
<function qr at 0x7feba447a9d8>
ipdb> pp args2
[array([], shape=(0, 3), dtype=float64)]
ipdb> up
> /home/mrocklin/workspace/dask/dask/local.py(296)execute_task()
    294     except BaseException as e:
    295         result = pack_exception(e, dumps)
--> 296         failed = True
    297     return key, result, failed
    298 

ipdb> pp key
('qr-bb4232513fb6d6a0b66ccbea15fe7885', 5, 0)

Metadata

Metadata

Assignees

No one assigned

    Labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions