Skip to content

Commit e453ffe

Browse files
koukou
committed
ARROW-15873: [CI] Migrate from Ubuntu 21.04 to 22.04
Ubuntu 21.04 reached EOL at 2022-01. Ubuntu 22.04 will be released in 2022-04. Closes #12781 from kou/ci-ubuntu-22.04 Authored-by: Sutou Kouhei <kou@clear-code.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent 961ec77 commit e453ffe

10 files changed

Lines changed: 227 additions & 103 deletions

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
!ci/**
2929
!c_glib/Gemfile
3030
!dev/archery/setup.py
31+
!dev/release/setup-*.sh
3132
!docs/requirements*.txt
3233
!python/requirements*.txt
3334
!python/manylinux1/**

‎.env‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,8 @@ GO=1.16
6161
HDFS=3.2.1
6262
JDK=8
6363
KARTOTHEK=latest
64-
LLVM=12
64+
# LLVM 12 and GCC 11 reports -Wmismatched-new-delete.
65+
LLVM=13
6566
MAVEN=3.5.4
6667
NODE=16
6768
NUMPY=latest

‎ci/docker/ubuntu-18.04-verify-rc.dockerfile‎

Lines changed: 3 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -19,42 +19,8 @@ ARG arch=amd64
1919
FROM ${arch}/ubuntu:18.04
2020

2121
ENV DEBIAN_FRONTEND=noninteractive
22-
23-
ARG llvm=12
24-
RUN apt-get update -y -q && \
25-
apt-get install -y -q --no-install-recommends \
26-
apt-transport-https \
27-
ca-certificates \
28-
gnupg \
29-
wget && \
30-
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
31-
echo "deb https://apt.llvm.org/bionic/ llvm-toolchain-bionic-${llvm} main" > \
32-
/etc/apt/sources.list.d/llvm.list && \
33-
apt-get update -y -q && \
34-
apt-get install -y -q --no-install-recommends \
35-
build-essential \
36-
clang \
37-
cmake \
38-
curl \
39-
git \
40-
libcurl4-openssl-dev \
41-
libgirepository1.0-dev \
42-
libglib2.0-dev \
43-
libsqlite3-dev \
44-
libssl-dev \
45-
llvm-${llvm}-dev \
46-
maven \
47-
ninja-build \
48-
openjdk-11-jdk \
49-
pkg-config \
50-
python3-pip \
51-
python3.8-dev \
52-
python3.8-venv \
53-
ruby-dev \
54-
wget \
55-
tzdata && \
22+
COPY dev/release/setup-ubuntu.sh /
23+
RUN /setup-ubuntu.sh && \
24+
rm /setup-ubuntu.sh && \
5625
apt-get clean && \
5726
rm -rf /var/lib/apt/lists*
58-
59-
RUN python3.8 -m pip install -U pip && \
60-
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1

‎ci/docker/ubuntu-20.04-verify-rc.dockerfile‎

Lines changed: 3 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,28 +19,8 @@ ARG arch=amd64
1919
FROM ${arch}/ubuntu:20.04
2020

2121
ENV DEBIAN_FRONTEND=noninteractive
22-
RUN apt-get update -y -q && \
23-
apt-get install -y -q --no-install-recommends \
24-
build-essential \
25-
clang \
26-
cmake \
27-
curl \
28-
git \
29-
libcurl4-openssl-dev \
30-
libgirepository1.0-dev \
31-
libglib2.0-dev \
32-
libsqlite3-dev \
33-
libssl-dev \
34-
llvm-dev \
35-
maven \
36-
ninja-build \
37-
nlohmann-json3-dev \
38-
openjdk-11-jdk \
39-
pkg-config \
40-
python3-dev \
41-
python3-pip \
42-
python3-venv \
43-
ruby-dev \
44-
wget && \
22+
COPY dev/release/setup-ubuntu.sh /
23+
RUN /setup-ubuntu.sh && \
24+
rm /setup-ubuntu.sh && \
4525
apt-get clean && \
4626
rm -rf /var/lib/apt/lists*

‎ci/docker/ubuntu-22.04-cpp-minimal.dockerfile‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
ARG base=amd64/ubuntu:22.04
19+
FROM ${base}
20+
21+
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
22+
23+
RUN echo "debconf debconf/frontend select Noninteractive" | \
24+
debconf-set-selections
25+
26+
RUN apt-get update -y -q && \
27+
apt-get install -y -q \
28+
build-essential \
29+
ccache \
30+
cmake \
31+
git \
32+
libssl-dev \
33+
libcurl4-openssl-dev \
34+
python3-pip \
35+
wget && \
36+
apt-get clean && \
37+
rm -rf /var/lib/apt/lists*
38+
39+
# Installs LLVM toolchain, for Gandiva and testing other compilers
40+
#
41+
# Note that this is installed before the base packages to improve iteration
42+
# while debugging package list with docker build.
43+
ARG llvm
44+
RUN latest_system_llvm=14 && \
45+
if [ ${llvm} -gt ${latest_system_llvm} ]; then \
46+
apt-get update -y -q && \
47+
apt-get install -y -q --no-install-recommends \
48+
apt-transport-https \
49+
ca-certificates \
50+
gnupg \
51+
lsb-release \
52+
wget && \
53+
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
54+
code_name=$(lsb_release --codename --short) && \
55+
if [ ${llvm} -gt 10 ]; then \
56+
echo "deb https://apt.llvm.org/${code_name}/ llvm-toolchain-${code_name}-${llvm} main" > \
57+
/etc/apt/sources.list.d/llvm.list; \
58+
fi; \
59+
fi && \
60+
apt-get update -y -q && \
61+
apt-get install -y -q --no-install-recommends \
62+
clang-${llvm} \
63+
llvm-${llvm}-dev && \
64+
apt-get clean && \
65+
rm -rf /var/lib/apt/lists*
66+
67+
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
68+
RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
69+
70+
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
71+
RUN /arrow/ci/scripts/install_gcs_testbench.sh default
72+
73+
ENV ARROW_BUILD_TESTS=ON \
74+
ARROW_DATASET=ON \
75+
ARROW_FLIGHT=ON \
76+
ARROW_GANDIVA=ON \
77+
ARROW_GCS=ON \
78+
ARROW_HDFS=ON \
79+
ARROW_HOME=/usr/local \
80+
ARROW_INSTALL_NAME_RPATH=OFF \
81+
ARROW_NO_DEPRECATED_API=ON \
82+
ARROW_ORC=ON \
83+
ARROW_PARQUET=ON \
84+
ARROW_PLASMA=ON \
85+
ARROW_S3=ON \
86+
ARROW_USE_CCACHE=ON \
87+
ARROW_WITH_BROTLI=ON \
88+
ARROW_WITH_BZ2=ON \
89+
ARROW_WITH_LZ4=ON \
90+
ARROW_WITH_OPENTELEMETRY=OFF \
91+
ARROW_WITH_SNAPPY=ON \
92+
ARROW_WITH_ZLIB=ON \
93+
ARROW_WITH_ZSTD=ON \
94+
CMAKE_GENERATOR="Unix Makefiles" \
95+
PARQUET_BUILD_EXAMPLES=ON \
96+
PARQUET_BUILD_EXECUTABLES=ON \
97+
PATH=/usr/lib/ccache/:$PATH \
98+
PYTHON=python3

ci/docker/ubuntu-21.04-cpp.dockerfile renamed to ci/docker/ubuntu-22.04-cpp.dockerfile

Lines changed: 35 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
ARG base=amd64/ubuntu:21.04
18+
ARG base=amd64/ubuntu:22.04
1919
FROM ${base}
2020

2121
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
@@ -29,7 +29,7 @@ RUN echo "debconf debconf/frontend select Noninteractive" | \
2929
# while debugging package list with docker build.
3030
ARG clang_tools
3131
ARG llvm
32-
RUN latest_system_llvm=12 && \
32+
RUN latest_system_llvm=14 && \
3333
if [ ${llvm} -gt ${latest_system_llvm} -o \
3434
${clang_tools} -gt ${latest_system_llvm} ]; then \
3535
apt-get update -y -q && \
@@ -96,6 +96,7 @@ RUN apt-get update -y -q && \
9696
pkg-config \
9797
protobuf-compiler \
9898
protobuf-compiler-grpc \
99+
python3-dev \
99100
python3-pip \
100101
rapidjson-dev \
101102
rsync \
@@ -104,6 +105,38 @@ RUN apt-get update -y -q && \
104105
apt-get clean && \
105106
rm -rf /var/lib/apt/lists*
106107

108+
ARG gcc_version=""
109+
RUN if [ "${gcc_version}" = "" ]; then \
110+
apt-get update -y -q && \
111+
apt-get install -y -q --no-install-recommends \
112+
g++ \
113+
gcc; \
114+
else \
115+
if [ "${gcc_version}" -gt "11" ]; then \
116+
apt-get update -y -q && \
117+
apt-get install -y -q --no-install-recommends software-properties-common && \
118+
add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
119+
fi; \
120+
apt-get update -y -q && \
121+
apt-get install -y -q --no-install-recommends \
122+
g++-${gcc_version} \
123+
gcc-${gcc_version} && \
124+
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
125+
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
126+
update-alternatives --install \
127+
/usr/bin/$(uname --machine)-linux-gnu-gcc \
128+
$(uname --machine)-linux-gnu-gcc \
129+
/usr/bin/$(uname --machine)-linux-gnu-gcc-${gcc_version} 100 && \
130+
update-alternatives --install \
131+
/usr/bin/$(uname --machine)-linux-gnu-g++ \
132+
$(uname --machine)-linux-gnu-g++ \
133+
/usr/bin/$(uname --machine)-linux-gnu-g++-${gcc_version} 100 && \
134+
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
135+
update-alternatives --set cc /usr/bin/gcc && \
136+
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \
137+
update-alternatives --set c++ /usr/bin/g++; \
138+
fi
139+
107140
COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
108141
RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
109142

@@ -149,27 +182,3 @@ ENV ARROW_BUILD_TESTS=ON \
149182
Protobuf_SOURCE=BUNDLED \
150183
PATH=/usr/lib/ccache/:$PATH \
151184
PYTHON=python3
152-
153-
ARG gcc_version=""
154-
RUN if [ "${gcc_version}" = "" ]; then \
155-
apt-get update -y -q && \
156-
apt-get install -y -q --no-install-recommends \
157-
g++ \
158-
gcc; \
159-
else \
160-
if [ "${gcc_version}" -gt "10" ]; then \
161-
apt-get update -y -q && \
162-
apt-get install -y -q --no-install-recommends software-properties-common && \
163-
add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
164-
fi; \
165-
apt-get update -y -q && \
166-
apt-get install -y -q --no-install-recommends \
167-
g++-${gcc_version} \
168-
gcc-${gcc_version} && \
169-
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
170-
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
171-
update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
172-
update-alternatives --set cc /usr/bin/gcc && \
173-
update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \
174-
update-alternatives --set c++ /usr/bin/g++; \
175-
fi

‎ci/docker/ubuntu-22.04-verify-rc.dockerfile‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
ARG arch=amd64
19+
FROM ${arch}/ubuntu:22.04
20+
21+
ENV DEBIAN_FRONTEND=noninteractive
22+
COPY dev/release/setup-ubuntu.sh /
23+
RUN /setup-ubuntu.sh && \
24+
rm /setup-ubuntu.sh && \
25+
apt-get clean && \
26+
rm -rf /var/lib/apt/lists*

‎dev/release/setup-ubuntu.sh‎

Lines changed: 48 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/bin/bash
2+
#
13
# Licensed to the Apache Software Foundation (ASF) under one
24
# or more contributor license agreements. See the NOTICE file
35
# distributed with this work for additional information
@@ -16,10 +18,40 @@
1618
# under the License.
1719

1820
# A script to install dependencies required for release
19-
# verification on Ubuntu 20.04
21+
# verification on Ubuntu.
22+
23+
set -exu
24+
25+
codename=$(. /etc/os-release && echo ${UBUNTU_CODENAME})
2026

21-
apt-get update
22-
apt-get -y install \
27+
case ${codename} in
28+
bionic)
29+
llvm=12
30+
nlohmann_json=
31+
python=3.8
32+
apt-get update -y -q
33+
apt-get install -y -q --no-install-recommends \
34+
apt-transport-https \
35+
ca-certificates \
36+
gnupg \
37+
wget
38+
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
39+
echo "deb https://apt.llvm.org/${codename}/ llvm-toolchain-${codename}-${llvm} main" > \
40+
/etc/apt/sources.list.d/llvm.list
41+
apt-get update -y -q
42+
apt-get install -y -q --no-install-recommends \
43+
llvm-${llvm}-dev
44+
;;
45+
*)
46+
nlohmann_json=3
47+
python=3
48+
apt-get update -y -q
49+
apt-get install -y -q --no-install-recommends \
50+
llvm-dev
51+
;;
52+
esac
53+
54+
apt-get install -y -q --no-install-recommends \
2355
build-essential \
2456
clang \
2557
cmake \
@@ -30,12 +62,22 @@ apt-get -y install \
3062
libglib2.0-dev \
3163
libsqlite3-dev \
3264
libssl-dev \
33-
llvm-dev \
3465
maven \
3566
ninja-build \
67+
nlohmann-json${nlohmann_json}-dev \
3668
openjdk-11-jdk \
3769
pkg-config \
3870
python3-pip \
39-
python3-venv \
71+
python${python}-dev \
72+
python${python}-venv \
4073
ruby-dev \
41-
wget
74+
wget \
75+
tzdata
76+
77+
case ${codename} in
78+
bionic)
79+
python${python} -m pip install -U pip
80+
update-alternatives \
81+
--install /usr/bin/python3 python3 /usr/bin/python${python} 1
82+
;;
83+
esac

0 commit comments

Comments
 (0)