[Release] Fix dask dependencies (#34261)

can-anyscale · web-flow · commit e3533efbc98b · 2023-04-12T11:22:54.000-07:00
Some if not all of the dask release tests are failing because of dependency hell. In short, boto, s3sf does not work well with boto3 that is installed in anyscale dataplane. Good news is these tests do not need these dependencies anyway (since anyscale already installed them properly). Related issue number Closes #19399
diff --git a/release/nightly_tests/chaos_test/dask_on_ray_app_config_reconstruction.yaml b/release/nightly_tests/chaos_test/dask_on_ray_app_config_reconstruction.yaml
@@ -3,11 +3,12 @@ env_vars: {"RAY_lineage_pinning_enabled": "1"}
 debian_packages: []
 
 python:
-  pip_packages: ["dask[complete]", tqdm, scipy, xarray, zarr, boto, s3fs, pyarrow]
+  pip_packages: ["dask[complete]", tqdm, scipy, xarray, zarr, pyarrow]
   conda_packages: []
 
 post_build_cmds:
   # - pip install fastparquet
+  - pip3 install boto3 s3fs
   - pip3 uninstall -y ray && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
   - pip3 install ray[default]
   - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}
diff --git a/release/nightly_tests/dask_on_ray/dask_on_ray_app_config.yaml b/release/nightly_tests/dask_on_ray/dask_on_ray_app_config.yaml
@@ -7,11 +7,12 @@ env_vars: {"RAY_worker_killing_policy": "retriable_lifo"}
 debian_packages: []
 
 python:
-  pip_packages: ["dask[complete]", tqdm, scipy, xarray, zarr, boto, s3fs, pyarrow, pytest]
+  pip_packages: ["dask[complete]", tqdm, scipy, xarray, zarr, pyarrow, pytest]
   conda_packages: []
 
 post_build_cmds:
   # - pip install fastparquet
+  - pip3 install boto3 s3fs
   - pip3 uninstall -y ray && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
   - pip3 install -U ray[default]
   - echo {{env["DATESTAMP"]}}
diff --git a/release/nightly_tests/dask_on_ray/large_scale_dask_on_ray_app_config.yaml b/release/nightly_tests/dask_on_ray/large_scale_dask_on_ray_app_config.yaml
@@ -2,13 +2,14 @@ base_image: {{ env["RAY_IMAGE_NIGHTLY_CPU"] | default("anyscale/ray:nightly-py37
 debian_packages: []
 
 python:
-  pip_packages: ["dask[complete]", tqdm, scipy, xarray, zarr, boto, s3fs, pyarrow, pytest]
+  pip_packages: ["dask[complete]", tqdm, scipy, xarray, zarr, pyarrow, pytest]
   conda_packages: []
 
 post_build_cmds:
   # - pip install fastparquet
+  - pip3 install boto3 s3fs
   - pip3 install -U pytest
-  -  pip3 uninstall -y ray && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
+  - pip3 uninstall -y ray && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }}
   - pip3 install -U ray[default]
   - echo {{env["DATESTAMP"]}}
   - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }}
