Ensure arrow FS tokenize

fjetter · fjetter · commit 6758d14f9049 · 2024-02-02T17:51:52.000+01:00
diff --git a/dask/dataframe/io/parquet/arrow.py b/dask/dataframe/io/parquet/arrow.py
@@ -11,6 +11,7 @@
 import numpy as np
 import pandas as pd
 import pyarrow as pa
+import pyarrow.fs as pa_fs
 import pyarrow.parquet as pq
 
 # Check PyArrow version for feature support
@@ -20,7 +21,7 @@
 from pyarrow import fs as pa_fs
 
 import dask
-from dask.base import tokenize
+from dask.base import normalize_token, tokenize
 from dask.core import flatten
 from dask.dataframe.backends import pyarrow_schema_dispatch
 from dask.dataframe.io.parquet.utils import (
@@ -55,6 +56,11 @@
 }
 
 
+@normalize_token.register(pa_fs.FileSystem)
+def tokenize_arrowfs(obj):
+    return obj.__reduce__()
+
+
 #
 #  Helper Utilities
 #
