Numpy 1.16.0 test failure, dtype construction seems wrong #4352
Description
Hello,
Debian is testing numpy 1.16.0 release candidates and a few test failures were noticed.
Here's a couple of them, I think there were 7 total of this type
______________________________ test_field_access _______________________________
def test_field_access():
x = np.array([(1, 1.0), (2, 2.0)], dtype=[('a', 'i4'), ('b', 'f4')])
y = from_array(x, chunks=(1,))
assert_eq(y['a'], x['a'])
> assert_eq(y[['b', 'a']], x[['b', 'a']])
/usr/lib/python3/dist-packages/dask/array/tests/test_array_core.py:826:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
a = array([(1., 1), (2., 2)],
dtype={'names':['b','a'], 'formats':['<f4','<i4'], 'offsets':[4,0], 'itemsize':8})
b = array([(1., 1), (2., 2)],
dtype={'names':['b','a'], 'formats':['<f4','<i4'], 'offsets':[4,0], 'itemsize':8})
check_shape = True, check_graph = True, kwargs = {}
a_original = dask.array<getitem, shape=(2,), dtype=[('b', '<f4'), ('a', '<i4')], chunksize=(1,)>
b_original = array([(1., 1), (2., 2)],
dtype={'names':['b','a'], 'formats':['<f4','<i4'], 'offsets':[4,0], 'itemsize':8})
adt = dtype([('b', '<f4'), ('a', '<i4')])
def assert_eq(a, b, check_shape=True, check_graph=True, **kwargs):
a_original = a
b_original = b
if isinstance(a, Array):
assert a.dtype is not None
adt = a.dtype
if check_graph:
_check_dsk(a.dask)
a = a.compute(scheduler='sync')
if hasattr(a, 'todense'):
a = a.todense()
if not hasattr(a, 'dtype'):
a = np.array(a, dtype='O')
if _not_empty(a):
> assert a.dtype == a_original.dtype
E AssertionError
/usr/lib/python3/dist-packages/dask/array/utils.py:90: AssertionError
_________________________ test_field_access_with_shape _________________________
def test_field_access_with_shape():
dtype = [('col1', ('f4', (3, 2))), ('col2', ('f4', 3))]
data = np.ones((100, 50), dtype=dtype)
x = da.from_array(data, 10)
assert_eq(x['col1'], data['col1'])
> assert_eq(x[['col1']], data[['col1']])
/usr/lib/python3/dist-packages/dask/array/tests/test_array_core.py:835:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
a = array([[([[1., 1.], [1., 1.], [1., 1.]],),
([[1., 1.], [1., 1.], [1., 1.]],),
([[1., 1.], [1., 1.], [1....], [1., 1.], [1., 1.]],)]],
dtype={'names':['col1'], 'formats':[('<f4', (3, 2))], 'offsets':[0], 'itemsize':36})
b = array([[([[1., 1.], [1., 1.], [1., 1.]],),
([[1., 1.], [1., 1.], [1., 1.]],),
([[1., 1.], [1., 1.], [1....], [1., 1.], [1., 1.]],)]],
dtype={'names':['col1'], 'formats':[('<f4', (3, 2))], 'offsets':[0], 'itemsize':36})
check_shape = True, check_graph = True, kwargs = {}
a_original = dask.array<getitem, shape=(100, 50), dtype=[('col1', '<f4', (3, 2))], chunksize=(10, 10)>
b_original = array([[([[1., 1.], [1., 1.], [1., 1.]],),
([[1., 1.], [1., 1.], [1., 1.]],),
([[1., 1.], [1., 1.], [1....], [1., 1.], [1., 1.]],)]],
dtype={'names':['col1'], 'formats':[('<f4', (3, 2))], 'offsets':[0], 'itemsize':36})
adt = dtype([('col1', '<f4', (3, 2))])
def assert_eq(a, b, check_shape=True, check_graph=True, **kwargs):
a_original = a
b_original = b
if isinstance(a, Array):
assert a.dtype is not None
adt = a.dtype
if check_graph:
_check_dsk(a.dask)
a = a.compute(scheduler='sync')
if hasattr(a, 'todense'):
a = a.todense()
if not hasattr(a, 'dtype'):
a = np.array(a, dtype='O')
if _not_empty(a):
> assert a.dtype == a_original.dtype
E AssertionError
/usr/lib/python3/dist-packages/dask/array/utils.py:90: AssertionError
(full CI log https://ci.debian.net/data/autopkgtest/testing/amd64/d/dask/1638550/log.gz )
and the debian dask CI page if you're interested https://ci.debian.net/packages/d/dask/testing/amd64/
I fixed most of the test failures with this patch which seems to match the function names a bit better.
--- a/dask/array/numpy_compat.py
+++ b/dask/array/numpy_compat.py
@@ -335,7 +335,7 @@
def _make_sliced_dtype(dtype, index):
- if LooseVersion(np.__version__) == LooseVersion("1.14.0"):
+ if LooseVersion(np.__version__) >= LooseVersion("1.14.0"):
return _make_sliced_dtype_np_ge_14(dtype, index)
else:
return _make_sliced_dtype_np_lt_14(dtype, index)
I can turn it into a pull request if you'd like.