Skip to content

Commit cc6e506

Browse files
jorisvandenbosschejorisvandenbossche
committed
Merge remote-tracking branch 'upstream/main' into fixup-tests
2 parents ed96326 + dd6d728 commit cc6e506

24 files changed

Lines changed: 318 additions & 217 deletions

‎.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ ULIMIT_CORE=-1
4848
# Default versions for platforms
4949
ALMALINUX=8
5050
ALPINE_LINUX=3.16
51-
DEBIAN=11
51+
DEBIAN=12
5252
FEDORA=39
5353
UBUNTU=20.04
5454

‎.github/workflows/go.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ jobs:
9090
echo "JSON" >> "$GITHUB_OUTPUT"
9191
9292
docker:
93-
name: ${{ matrix.arch-label }} Debian 11 Go ${{ matrix.go }}
93+
name: ${{ matrix.arch-label }} Debian 12 Go ${{ matrix.go }}
9494
needs: docker-targets
9595
runs-on: ${{ matrix.runs-on }}
9696
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
@@ -154,7 +154,7 @@ jobs:
154154
CONBENCH_EMAIL: ${{ secrets.CONBENCH_EMAIL }}
155155
CONBENCH_PASSWORD: ${{ secrets.CONBENCH_PASS }}
156156
CONBENCH_REF: ${{ github.ref_name }}
157-
CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-11
157+
CONBENCH_MACHINE_INFO_NAME: ${{ matrix.arch }}-debian-12
158158
run: |
159159
python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
160160
python3 ci/scripts/go_bench_adapt.py
@@ -181,7 +181,7 @@ jobs:
181181
GOARCH=386 go build ./...
182182
183183
docker_cgo:
184-
name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO
184+
name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO
185185
runs-on: ubuntu-latest
186186
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
187187
timeout-minutes: 20
@@ -222,7 +222,7 @@ jobs:
222222

223223

224224
docker_cgo_python:
225-
name: AMD64 Debian 11 Go ${{ matrix.go }} - CGO Python
225+
name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO Python
226226
runs-on: ubuntu-latest
227227
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
228228
timeout-minutes: 15

‎.github/workflows/js.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ permissions:
4141
jobs:
4242

4343
docker:
44-
name: AMD64 Debian 11 NodeJS 18
44+
name: AMD64 Debian 12 NodeJS 18
4545
runs-on: ubuntu-latest
4646
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
4747
timeout-minutes: 60

ci/docker/debian-11-cpp.dockerfile renamed to ci/docker/debian-12-cpp.dockerfile

Lines changed: 39 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -16,46 +16,42 @@
1616
# under the License.
1717

1818
ARG arch=amd64
19-
FROM ${arch}/debian:11
19+
FROM ${arch}/debian:12
2020
ARG arch
2121

2222
ENV DEBIAN_FRONTEND noninteractive
2323

2424
ARG llvm
25-
# We can't use LLVM 14 or later from apt.llvm.org on i386 because LLVM
26-
# 14 or later dropped support for i386.
2725
RUN apt-get update -y -q && \
28-
apt-get install -y -q --no-install-recommends \
29-
dpkg-dev && \
30-
latest_available_llvm_i386=13 && \
31-
if [ $(dpkg-architecture -qDEB_HOST_ARCH) = "i386" -a \
32-
"${llvm}" -gt "${latest_available_llvm_i386}" ]; then \
33-
available_llvm="${latest_available_llvm_i386}"; \
34-
else \
35-
available_llvm="${llvm}"; \
36-
fi && \
37-
apt-get update -y -q && \
3826
apt-get install -y -q --no-install-recommends \
3927
apt-transport-https \
4028
ca-certificates \
4129
gnupg \
30+
lsb-release \
4231
wget && \
43-
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
44-
echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${available_llvm} main" > \
45-
/etc/apt/sources.list.d/llvm.list && \
32+
if [ ${llvm} -ge 17 ]; then \
33+
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | \
34+
gpg \
35+
--import - \
36+
--keyring /usr/share/keyrings/llvm-snapshot.gpg \
37+
--no-default-keyring && \
38+
echo "deb[keyring=/usr/share/keyrings/llvm-snapshot.gpg] https://apt.llvm.org/$(lsb_release --codename --short)/ llvm-toolchain-$(lsb_release --codename --short)-${available_llvm} main" > \
39+
/etc/apt/sources.list.d/llvm.list; \
40+
fi && \
4641
apt-get update -y -q && \
4742
apt-get install -y -q --no-install-recommends \
4843
autoconf \
4944
ccache \
50-
clang-${available_llvm} \
45+
clang-${llvm} \
5146
cmake \
5247
curl \
5348
g++ \
5449
gcc \
5550
gdb \
5651
git \
5752
libbenchmark-dev \
58-
libboost-all-dev \
53+
libboost-filesystem-dev \
54+
libboost-system-dev \
5955
libbrotli-dev \
6056
libbz2-dev \
6157
libc-ares-dev \
@@ -64,22 +60,35 @@ RUN apt-get update -y -q && \
6460
libgmock-dev \
6561
libgoogle-glog-dev \
6662
libgrpc++-dev \
63+
libidn2-dev \
64+
libkrb5-dev \
65+
libldap-dev \
6766
liblz4-dev \
67+
libnghttp2-dev \
6868
libprotobuf-dev \
6969
libprotoc-dev \
70+
libpsl-dev \
7071
libre2-dev \
72+
librtmp-dev \
7173
libsnappy-dev \
74+
libsqlite3-dev \
75+
libssh-dev \
76+
libssh2-1-dev \
7277
libssl-dev \
7378
libthrift-dev \
7479
libutf8proc-dev \
80+
libxml2-dev \
7581
libzstd-dev \
76-
llvm-${available_llvm}-dev \
82+
llvm-${llvm}-dev \
7783
make \
7884
ninja-build \
7985
nlohmann-json3-dev \
86+
npm \
8087
pkg-config \
8188
protobuf-compiler-grpc \
89+
python3-dev \
8290
python3-pip \
91+
python3-venv \
8392
rapidjson-dev \
8493
rsync \
8594
tzdata \
@@ -93,18 +102,24 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
93102
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
94103
RUN /arrow/ci/scripts/install_gcs_testbench.sh default
95104

105+
COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
106+
RUN /arrow/ci/scripts/install_azurite.sh
107+
96108
COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
97109
RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
98110

99-
ENV absl_SOURCE=BUNDLED \
100-
ARROW_ACERO=ON \
111+
ENV ARROW_ACERO=ON \
112+
ARROW_AZURE=ON \
101113
ARROW_BUILD_TESTS=ON \
102114
ARROW_DATASET=ON \
103115
ARROW_DEPENDENCY_SOURCE=SYSTEM \
116+
ARROW_DATASET=ON \
104117
ARROW_FLIGHT=ON \
118+
ARROW_FLIGHT_SQL=ON \
105119
ARROW_GANDIVA=ON \
106120
ARROW_GCS=ON \
107121
ARROW_HOME=/usr/local \
122+
ARROW_NO_DEPRECATED_API=ON \
108123
ARROW_ORC=ON \
109124
ARROW_PARQUET=ON \
110125
ARROW_S3=ON \
@@ -113,15 +128,14 @@ ENV absl_SOURCE=BUNDLED \
113128
ARROW_WITH_BROTLI=ON \
114129
ARROW_WITH_BZ2=ON \
115130
ARROW_WITH_LZ4=ON \
116-
ARROW_WITH_OPENTELEMETRY=OFF \
131+
ARROW_WITH_OPENTELEMETRY=ON \
117132
ARROW_WITH_SNAPPY=ON \
118133
ARROW_WITH_ZLIB=ON \
119134
ARROW_WITH_ZSTD=ON \
120135
AWSSDK_SOURCE=BUNDLED \
121-
CC=gcc \
122-
CXX=g++ \
136+
Azure_SOURCE=BUNDLED \
123137
google_cloud_cpp_storage_SOURCE=BUNDLED \
124-
GTest_SOURCE=BUNDLED \
125138
ORC_SOURCE=BUNDLED \
126139
PATH=/usr/lib/ccache/:$PATH \
140+
PYTHON=python3 \
127141
xsimd_SOURCE=BUNDLED

ci/docker/debian-11-go-cgo-python.dockerfile renamed to ci/docker/debian-12-go-cgo-python.dockerfile

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,11 @@ ENV DEBIAN_FRONTEND noninteractive
2424
RUN apt-get update -y -q && \
2525
apt-get install -y -q --no-install-recommends \
2626
python3 \
27-
python3-pip && \
27+
python3-pip \
28+
python3-venv && \
2829
apt-get clean
2930

30-
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
31-
ln -s /usr/bin/pip3 /usr/local/bin/pip
32-
33-
# Need a newer pip than Debian's to install manylinux201x wheels
34-
RUN pip install -U pip
35-
36-
RUN pip install pyarrow cffi --only-binary pyarrow
31+
ENV ARROW_PYTHON_VENV /arrow-dev
32+
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
33+
. ${ARROW_PYTHON_VENV}/bin/activate && \
34+
pip install pyarrow cffi --only-binary pyarrow

ci/docker/debian-11-go.dockerfile renamed to ci/docker/debian-12-go.dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
ARG arch=amd64
1919
ARG go=1.19
2020
ARG staticcheck=v0.4.5
21-
FROM ${arch}/golang:${go}-bullseye
21+
FROM ${arch}/golang:${go}-bookworm
2222

2323
# FROM collects all the args, get back the staticcheck version arg
2424
ARG staticcheck

ci/docker/debian-11-js.dockerfile renamed to ci/docker/debian-12-js.dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
# under the License.
1717

1818
ARG arch=amd64
19-
ARG node=16
19+
ARG node=18
2020
FROM ${arch}/node:${node}
2121

2222
ENV NODE_NO_WARNINGS=1

‎ci/docker/linux-apt-c-glib.dockerfile‎

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ RUN apt-get update -y -q && \
2525
libglib2.0-doc \
2626
lsb-release \
2727
luarocks \
28+
meson \
2829
ninja-build \
2930
pkg-config \
3031
python3 \
@@ -36,24 +37,7 @@ RUN apt-get update -y -q && \
3637

3738
RUN luarocks install lgi
3839

39-
# pip on Ubuntu 20.04 may be buggy:
40-
#
41-
# Collecting meson
42-
# Downloading meson-0.53.2.tar.gz (1.6 MB)
43-
# Installing build dependencies: started
44-
# Installing build dependencies: finished with status 'done'
45-
# Getting requirements to build wheel: started
46-
# Getting requirements to build wheel: finished with status 'error'
47-
# ERROR: Command errored out with exit status 1:
48-
# command: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay
49-
# cwd: /tmp/pip-install-jn79a_kh/meson
50-
# Complete output (1 lines):
51-
# /usr/bin/python3: can't find '__main__' module in '/usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py'
52-
# ----------------------------------------
53-
# ERROR: Command errored out with exit status 1: /usr/bin/python3 /usr/share/python-wheels/pep517-0.7.0-py2.py3-none-any.whl/pep517/_in_process.py get_requires_for_build_wheel /tmp/tmpsk4jveay Check the logs for full command output.
54-
RUN (python3 -m pip install meson || \
55-
python3 -m pip install --no-use-pep517 meson) && \
56-
gem install --no-document bundler
40+
RUN gem install --no-document bundler
5741

5842
COPY c_glib/Gemfile /arrow/c_glib/
5943
RUN bundle install --gemfile /arrow/c_glib/Gemfile

‎ci/docker/linux-apt-python-3.dockerfile‎

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -18,26 +18,17 @@
1818
ARG base
1919
FROM ${base}
2020

21-
RUN apt-get update -y -q && \
22-
apt-get install -y -q \
23-
python3 \
24-
python3-pip \
25-
python3-dev && \
26-
apt-get clean && \
27-
rm -rf /var/lib/apt/lists/*
28-
29-
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
30-
ln -s /usr/bin/pip3 /usr/local/bin/pip
31-
32-
RUN pip install -U pip setuptools wheel
33-
3421
COPY python/requirements-build.txt \
3522
python/requirements-test.txt \
3623
/arrow/python/
3724

38-
RUN pip install \
39-
-r arrow/python/requirements-build.txt \
40-
-r arrow/python/requirements-test.txt
25+
ENV ARROW_PYTHON_VENV /arrow-dev
26+
RUN python3 -m venv ${ARROW_PYTHON_VENV} && \
27+
. ${ARROW_PYTHON_VENV}/bin/activate && \
28+
pip install -U pip setuptools wheel && \
29+
pip install \
30+
-r arrow/python/requirements-build.txt \
31+
-r arrow/python/requirements-test.txt
4132

4233
ARG numba
4334
COPY ci/scripts/install_numba.sh /arrow/ci/scripts/

‎ci/scripts/go_cgo_python_test.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ set -ex
2121

2222
source_dir=${1}/go
2323

24+
if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
25+
. "${ARROW_PYTHON_VENV}/bin/activate"
26+
fi
27+
2428
export GOFLAGS="${GOFLAGS} -gcflags=all=-d=checkptr"
2529

2630
pushd ${source_dir}/arrow/cdata/test

0 commit comments

Comments
 (0)