Skip to content

Zarr 2.18 Breaks ParticleSet.execute(): ValueError: buffer source array is read-only #1564

@jamespolly

Description

@jamespolly

The latest release of Zarr is v2.18.0 and was released last week.

Using the parcels development environment, the following error is generated when performing particleset.execute().

Rather than include some other links pointing to possibly unrelated instances of this error, I'll leave the traceback below, and say that pinning the zarr dependency to zarr<2.18.0 in environment.yml will fix this.

Zarr v3.0.0 is soon in the works for this summer and maybe this behavior will change.

INFO: Output files are stored in [...]out.zarr.
  0%|                                                                                                                                                          | 0/18000.0 [00:00<?, ?it/s]---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
File ~/[...]:121
--> 121 pset.execute(
    122     kernels,
    123     runtime=timedelta(hours=5),
    124     dt=timedelta(minutes=5),
    125     output_file=output_file,
    126 )

File ~/repos/parcels/parcels/particleset.py:988, in ParticleSet.execute(self, pyfunc, pyfunc_inter, endtime, runtime, dt, output_file, verbose_progress, postIterationCallbacks, callbackdt, delete_cfiles)
    986         output_file.write_latest_locations(self, time)
    987     else:
--> 988         output_file.write(self, time_at_startofloop)
    989 if np.isfinite(outputdt):
    990     next_output += outputdt * np.sign(dt)

File ~/repos/parcels/parcels/particlefile.py:273, in ParticleFile.write(self, pset, time, indices)
    271         else:
    272             if max(obs) >= Z[varout].shape[1]:
--> 273                 self._extend_zarr_dims(Z[varout], store, dtype=self.vars_to_write[var], axis=1)
    274             Z[varout].vindex[ids, obs] = pset.particledata.getvardata(var, indices_to_write)
    276 pset.particledata.setvardata('obs_written', indices_to_write, obs+1)

File ~/repos/parcels/parcels/particlefile.py:184, in ParticleFile._extend_zarr_dims(self, Z, store, dtype, axis)
    182     obs = zarr.group(store=store, overwrite=False)["obs"]
    183     if len(obs) == Z.shape[1]:
--> 184         obs.append(np.arange(self.chunks[1])+obs[-1]+1)
    185 else:
    186     extra_trajs = self.maxids - Z.shape[0]

File ~/.conda/envs/parcels/lib/python3.12/site-packages/zarr/core.py:800, in Array.__getitem__(self, selection)
    798     result = self.get_orthogonal_selection(pure_selection, fields=fields)
    799 else:
--> 800     result = self.get_basic_selection(pure_selection, fields=fields)
    801 return result

File ~/.conda/envs/parcels/lib/python3.12/site-packages/zarr/core.py:926, in Array.get_basic_selection(self, selection, out, fields)
    924     return self._get_basic_selection_zd(selection=selection, out=out, fields=fields)
    925 else:
--> 926     return self._get_basic_selection_nd(selection=selection, out=out, fields=fields)

File ~/.conda/envs/parcels/lib/python3.12/site-packages/zarr/core.py:968, in Array._get_basic_selection_nd(self, selection, out, fields)
    962 def _get_basic_selection_nd(self, selection, out=None, fields=None):
    963     # implementation of basic selection for array with at least one dimension
    964 
    965     # setup indexer
    966     indexer = BasicIndexer(selection, self)
--> 968     return self._get_selection(indexer=indexer, out=out, fields=fields)

File ~/.conda/envs/parcels/lib/python3.12/site-packages/zarr/core.py:1343, in Array._get_selection(self, indexer, out, fields)
   1340 if math.prod(out_shape) > 0:
   1341     # allow storage to get multiple items at once
   1342     lchunk_coords, lchunk_selection, lout_selection = zip(*indexer)
-> 1343     self._chunk_getitems(
   1344         lchunk_coords,
   1345         lchunk_selection,
   1346         out,
   1347         lout_selection,
   1348         drop_axes=indexer.drop_axes,
   1349         fields=fields,
   1350     )
   1351 if out.shape:
   1352     return out

File ~/.conda/envs/parcels/lib/python3.12/site-packages/zarr/core.py:2181, in Array._chunk_getitems(self, lchunk_coords, lchunk_selection, out, lout_selection, drop_axes, fields)
   2179 for ckey, chunk_select, out_select in zip(ckeys, lchunk_selection, lout_selection):
   2180     if ckey in cdatas:
-> 2181         self._process_chunk(
   2182             out,
   2183             cdatas[ckey],
   2184             chunk_select,
   2185             drop_axes,
   2186             out_is_ndarray,
   2187             fields,
   2188             out_select,
   2189             partial_read_decode=partial_read_decode,
   2190         )
   2191     else:
   2192         # check exception type
   2193         if self._fill_value is not None:

File ~/.conda/envs/parcels/lib/python3.12/site-packages/zarr/core.py:2049, in Array._process_chunk(self, out, cdata, chunk_selection, drop_axes, out_is_ndarray, fields, out_selection, partial_read_decode)
   2047     if isinstance(cdata, PartialReadBuffer):
   2048         cdata = cdata.read_full()
-> 2049     self._compressor.decode(cdata, dest)
   2050 else:
   2051     if isinstance(cdata, UncompressedPartialReadBufferV3):

File numcodecs/blosc.pyx:564, in numcodecs.blosc.Blosc.decode()

File numcodecs/blosc.pyx:365, in numcodecs.blosc.decompress()

File numcodecs/compat_ext.pyx:16, in numcodecs.compat_ext.Buffer.__cinit__()

ValueError: buffer source array is read-only

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions