File tree Expand file tree Collapse file tree
Expand file tree Collapse file tree Original file line number Diff line number Diff line change 11from __future__ import annotations
22
3+ from distributed .shuffle ._arrow import check_minimal_arrow_version
34from distributed .shuffle ._scheduler_extension import ShuffleSchedulerExtension
45from distributed .shuffle ._shuffle import P2PShuffleLayer , rearrange_by_column_p2p
56from distributed .shuffle ._worker_extension import ShuffleWorkerExtension
67
78__all__ = [
9+ "check_minimal_arrow_version" ,
810 "P2PShuffleLayer" ,
911 "rearrange_by_column_p2p" ,
1012 "ShuffleSchedulerExtension" ,
Original file line number Diff line number Diff line change @@ -29,6 +29,12 @@ def check_dtype_support(meta_input: pd.DataFrame) -> None:
2929
3030
3131def check_minimal_arrow_version () -> None :
32+ """Verify that the the correct version of pyarrow is installed to support
33+ the P2P extension.
34+
35+ Raises a RuntimeError in case pyarrow is not installed or installed version
36+ is not recent enough.
37+ """
3238 suffix = ""
3339 # First version to introduce Table.sort_by
3440 minversion = "7.0.0"
Original file line number Diff line number Diff line change 77from dask .highlevelgraph import HighLevelGraph
88from dask .layers import SimpleShuffleLayer
99
10- from distributed .shuffle ._arrow import check_dtype_support
11- from distributed .shuffle ._arrow import check_minimal_arrow_version
10+ from distributed .shuffle ._arrow import check_dtype_support , check_minimal_arrow_version
1211
1312logger = logging .getLogger ("distributed.shuffle" )
1413if TYPE_CHECKING :
You can’t perform that action at this time.
0 commit comments