Skip to content

Commit 92682f0

Browse files
jduojduo
authored
GH-39001: [Java] Modularize remaining modules (#39221)
### Rationale for this change Modularize remaining modules outside of memory modules, vector, and format. ### What changes are included in this PR? ### Are these changes tested? Yes, existing unit tests now run with modules when using JDK9+. ### Are there any user-facing changes? Yes. There are new command-line options that may be necessary. The way of specifying the output directory for JNI native library builds differs. The flight-grpc module has been eliminated since it is now built into flight-core. Documentation has been updated for these changes. **This PR includes breaking changes to public APIs.** There are a number of package structure changes and some modules now need additional command-line arguments. * Closes: #39001 Authored-by: James Duong <james.duong@improving.com> Signed-off-by: David Li <li.davidm96@gmail.com>
1 parent 143a7da commit 92682f0

98 files changed

Lines changed: 763 additions & 355 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎ci/scripts/integration_arrow_build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ if [ "${ARROW_INTEGRATION_JAVA}" == "ON" ]; then
4646
export ARROW_JAVA_CDATA="ON"
4747
export JAVA_JNI_CMAKE_ARGS="-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF -DARROW_JAVA_JNI_ENABLE_C=ON"
4848

49-
${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java/$(arch)
49+
${arrow_dir}/ci/scripts/java_jni_build.sh ${arrow_dir} ${ARROW_HOME} ${build_dir} /tmp/dist/java
5050
${arrow_dir}/ci/scripts/java_build.sh ${arrow_dir} ${build_dir} /tmp/dist/java
5151
fi
5252

‎ci/scripts/java_jni_build.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ arrow_install_dir=${2}
2424
build_dir=${3}/java_jni
2525
# The directory where the final binaries will be stored when scripts finish
2626
dist_dir=${4}
27-
2827
prefix_dir="${build_dir}/java-jni"
2928

3029
echo "=== Clear output directories and leftovers ==="
@@ -56,7 +55,6 @@ cmake \
5655
-DBUILD_TESTING=${ARROW_JAVA_BUILD_TESTS} \
5756
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
5857
-DCMAKE_PREFIX_PATH=${arrow_install_dir} \
59-
-DCMAKE_INSTALL_LIBDIR=lib \
6058
-DCMAKE_INSTALL_PREFIX=${prefix_dir} \
6159
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
6260
-DProtobuf_USE_STATIC_LIBS=ON \

‎ci/scripts/java_jni_macos_build.sh‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ case ${normalized_arch} in
3131
;;
3232
esac
3333
# The directory where the final binaries will be stored when scripts finish
34-
dist_dir=${3}/${normalized_arch}
34+
dist_dir=${3}
3535

3636
echo "=== Clear output directories and leftovers ==="
3737
# Clear output directories and leftovers
@@ -82,7 +82,6 @@ cmake \
8282
-DARROW_S3=${ARROW_S3} \
8383
-DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
8484
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
85-
-DCMAKE_INSTALL_LIBDIR=lib \
8685
-DCMAKE_INSTALL_PREFIX=${install_dir} \
8786
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
8887
-DGTest_SOURCE=BUNDLED \
@@ -138,8 +137,8 @@ archery linking check-dependencies \
138137
--allow libncurses \
139138
--allow libobjc \
140139
--allow libz \
141-
libarrow_cdata_jni.dylib \
142-
libarrow_dataset_jni.dylib \
143-
libarrow_orc_jni.dylib \
144-
libgandiva_jni.dylib
140+
arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.dylib \
141+
arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.dylib \
142+
arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.dylib \
143+
gandiva_jni/${normalized_arch}/libgandiva_jni.dylib
145144
popd

‎ci/scripts/java_jni_manylinux_build.sh‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ case ${normalized_arch} in
2828
;;
2929
esac
3030
# The directory where the final binaries will be stored when scripts finish
31-
dist_dir=${3}/${normalized_arch}
31+
dist_dir=${3}
3232

3333
echo "=== Clear output directories and leftovers ==="
3434
# Clear output directories and leftovers
@@ -91,7 +91,6 @@ cmake \
9191
-DARROW_S3=${ARROW_S3} \
9292
-DARROW_USE_CCACHE=${ARROW_USE_CCACHE} \
9393
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
94-
-DCMAKE_INSTALL_LIBDIR=lib \
9594
-DCMAKE_INSTALL_PREFIX=${ARROW_HOME} \
9695
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
9796
-DGTest_SOURCE=BUNDLED \
@@ -164,8 +163,8 @@ archery linking check-dependencies \
164163
--allow libstdc++ \
165164
--allow libz \
166165
--allow linux-vdso \
167-
libarrow_cdata_jni.so \
168-
libarrow_dataset_jni.so \
169-
libarrow_orc_jni.so \
170-
libgandiva_jni.so
166+
arrow_cdata_jni/${normalized_arch}/libarrow_cdata_jni.so \
167+
arrow_dataset_jni/${normalized_arch}/libarrow_dataset_jni.so \
168+
arrow_orc_jni/${normalized_arch}/libarrow_orc_jni.so \
169+
gandiva_jni/${normalized_arch}/libgandiva_jni.so
171170
popd

‎ci/scripts/java_jni_windows_build.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ set -ex
2222
arrow_dir=${1}
2323
build_dir=${2}
2424
# The directory where the final binaries will be stored when scripts finish
25-
dist_dir=${3}/x86_64
25+
dist_dir=${3}
2626

2727
echo "=== Clear output directories and leftovers ==="
2828
# Clear output directories and leftovers
@@ -72,7 +72,6 @@ cmake \
7272
-DARROW_WITH_SNAPPY=ON \
7373
-DARROW_WITH_ZSTD=ON \
7474
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
75-
-DCMAKE_INSTALL_LIBDIR=lib \
7675
-DCMAKE_INSTALL_PREFIX=${install_dir} \
7776
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD} \
7877
-GNinja \

‎dev/archery/archery/integration/tester_java.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,8 @@ def __init__(self, *args, **kwargs):
259259
self._java_opts.append(
260260
'--add-opens=java.base/java.nio='
261261
'org.apache.arrow.memory.core,ALL-UNNAMED')
262+
self._java_opts.append(
263+
'--add-reads=org.apache.arrow.flight.core=ALL-UNNAMED')
262264

263265
def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
264266
cmd = (

‎dev/tasks/java-jars/github.yml‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -208,29 +208,29 @@ jobs:
208208
run: |
209209
set -x
210210
211-
test -f arrow/java-dist/x86_64/libarrow_cdata_jni.so
212-
test -f arrow/java-dist/x86_64/libarrow_dataset_jni.so
213-
test -f arrow/java-dist/x86_64/libarrow_orc_jni.so
214-
test -f arrow/java-dist/x86_64/libgandiva_jni.so
211+
test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.so
212+
test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.so
213+
test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.so
214+
test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.so
215215
216-
test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.so
217-
test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.so
218-
test -f arrow/java-dist/aarch_64/libarrow_orc_jni.so
219-
test -f arrow/java-dist/aarch_64/libgandiva_jni.so
216+
test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.so
217+
test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.so
218+
test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.so
219+
test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.so
220220
221-
test -f arrow/java-dist/x86_64/libarrow_cdata_jni.dylib
222-
test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib
223-
test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib
224-
test -f arrow/java-dist/x86_64/libgandiva_jni.dylib
221+
test -f arrow/java-dist/arrow_cdata_jni/x86_64/libarrow_cdata_jni.dylib
222+
test -f arrow/java-dist/arrow_dataset_jni/x86_64/libarrow_dataset_jni.dylib
223+
test -f arrow/java-dist/arrow_orc_jni/x86_64/libarrow_orc_jni.dylib
224+
test -f arrow/java-dist/gandiva_jni/x86_64/libgandiva_jni.dylib
225225
226-
test -f arrow/java-dist/aarch_64/libarrow_cdata_jni.dylib
227-
test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.dylib
228-
test -f arrow/java-dist/aarch_64/libarrow_orc_jni.dylib
229-
test -f arrow/java-dist/aarch_64/libgandiva_jni.dylib
226+
test -f arrow/java-dist/arrow_cdata_jni/aarch_64/libarrow_cdata_jni.dylib
227+
test -f arrow/java-dist/arrow_dataset_jni/aarch_64/libarrow_dataset_jni.dylib
228+
test -f arrow/java-dist/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
229+
test -f arrow/java-dist/gandiva_jni/aarch_64/libgandiva_jni.dylib
230230
231-
test -f arrow/java-dist/x86_64/arrow_cdata_jni.dll
232-
test -f arrow/java-dist/x86_64/arrow_dataset_jni.dll
233-
test -f arrow/java-dist/x86_64/arrow_orc_jni.dll
231+
test -f arrow/java-dist/arrow_cdata_jni/x86_64/arrow_cdata_jni.dll
232+
test -f arrow/java-dist/arrow_dataset_jni/x86_64/arrow_dataset_jni.dll
233+
test -f arrow/java-dist/arrow_orc_jni/x86_64/arrow_orc_jni.dll
234234
- name: Build bundled jar
235235
run: |
236236
set -e

‎dev/tasks/tasks.yml‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -810,13 +810,6 @@ tasks:
810810
- flight-core-{no_rc_snapshot_version}-tests.jar
811811
- flight-core-{no_rc_snapshot_version}.jar
812812
- flight-core-{no_rc_snapshot_version}.pom
813-
- flight-grpc-{no_rc_snapshot_version}-cyclonedx.json
814-
- flight-grpc-{no_rc_snapshot_version}-cyclonedx.xml
815-
- flight-grpc-{no_rc_snapshot_version}-javadoc.jar
816-
- flight-grpc-{no_rc_snapshot_version}-sources.jar
817-
- flight-grpc-{no_rc_snapshot_version}-tests.jar
818-
- flight-grpc-{no_rc_snapshot_version}.jar
819-
- flight-grpc-{no_rc_snapshot_version}.pom
820813
- flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.json
821814
- flight-integration-tests-{no_rc_snapshot_version}-cyclonedx.xml
822815
- flight-integration-tests-{no_rc_snapshot_version}-jar-with-dependencies.jar

‎docker-compose.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1343,7 +1343,7 @@ services:
13431343
command:
13441344
[ "/arrow/ci/scripts/cpp_build.sh /arrow /build &&
13451345
/arrow/ci/scripts/python_build.sh /arrow /build &&
1346-
/arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/$$(arch) &&
1346+
/arrow/ci/scripts/java_jni_build.sh /arrow $${ARROW_HOME} /build /tmp/dist/java/ &&
13471347
/arrow/ci/scripts/java_build.sh /arrow /build /tmp/dist/java &&
13481348
/arrow/ci/scripts/java_cdata_integration.sh /arrow /tmp/dist/java" ]
13491349

‎docs/source/developers/java/building.rst‎

Lines changed: 23 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -115,18 +115,17 @@ Maven
115115
$ export JAVA_HOME=<absolute path to your java home>
116116
$ java --version
117117
$ mvn generate-resources -Pgenerate-libs-cdata-all-os -N
118-
$ ls -latr ../java-dist/lib/<your system's architecture>
119-
|__ libarrow_cdata_jni.dylib
120-
|__ libarrow_cdata_jni.so
118+
$ ls -latr ../java-dist/lib
119+
|__ arrow_cdata_jni/
121120
122121
- To build only the JNI C Data Interface library (Windows):
123122

124123
.. code-block::
125124
126125
$ cd arrow/java
127126
$ mvn generate-resources -Pgenerate-libs-cdata-all-os -N
128-
$ dir "../java-dist/bin/x86_64"
129-
|__ arrow_cdata_jni.dll
127+
$ dir "../java-dist/bin"
128+
|__ arrow_cdata_jni/
130129
131130
- To build all JNI libraries (macOS / Linux) except the JNI C Data Interface library:
132131

@@ -136,19 +135,19 @@ Maven
136135
$ export JAVA_HOME=<absolute path to your java home>
137136
$ java --version
138137
$ mvn generate-resources -Pgenerate-libs-jni-macos-linux -N
139-
$ ls -latr java-dist/lib/<your system's architecture>/*_{jni,java}.*
140-
|__ libarrow_dataset_jni.dylib
141-
|__ libarrow_orc_jni.dylib
142-
|__ libgandiva_jni.dylib
138+
$ ls -latr java-dist/lib
139+
|__ arrow_dataset_jni/
140+
|__ arrow_orc_jni/
141+
|__ gandiva_jni/
143142
144143
- To build all JNI libraries (Windows) except the JNI C Data Interface library:
145144

146145
.. code-block::
147146
148147
$ cd arrow/java
149148
$ mvn generate-resources -Pgenerate-libs-jni-windows -N
150-
$ dir "../java-dist/bin/x86_64"
151-
|__ arrow_dataset_jni.dll
149+
$ dir "../java-dist/bin"
150+
|__ arrow_dataset_jni/
152151
153152
CMake
154153
~~~~~
@@ -166,12 +165,10 @@ CMake
166165
-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF \
167166
-DBUILD_TESTING=OFF \
168167
-DCMAKE_BUILD_TYPE=Release \
169-
-DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
170168
-DCMAKE_INSTALL_PREFIX=java-dist
171169
$ cmake --build java-cdata --target install --config Release
172170
$ ls -latr java-dist/lib
173-
|__ libarrow_cdata_jni.dylib
174-
|__ libarrow_cdata_jni.so
171+
|__ arrow_cdata_jni/
175172
176173
- To build only the JNI C Data Interface library (Windows):
177174

@@ -186,11 +183,10 @@ CMake
186183
-DARROW_JAVA_JNI_ENABLE_DEFAULT=OFF ^
187184
-DBUILD_TESTING=OFF ^
188185
-DCMAKE_BUILD_TYPE=Release ^
189-
-DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
190186
-DCMAKE_INSTALL_PREFIX=java-dist
191187
$ cmake --build java-cdata --target install --config Release
192188
$ dir "java-dist/bin"
193-
|__ arrow_cdata_jni.dll
189+
|__ arrow_cdata_jni/
194190
195191
- To build all JNI libraries (macOS / Linux) except the JNI C Data Interface library:
196192

@@ -222,7 +218,6 @@ CMake
222218
-DARROW_SUBSTRAIT=ON \
223219
-DARROW_USE_CCACHE=ON \
224220
-DCMAKE_BUILD_TYPE=Release \
225-
-DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
226221
-DCMAKE_INSTALL_PREFIX=java-dist \
227222
-DCMAKE_UNITY_BUILD=ON
228223
$ cmake --build cpp-jni --target install --config Release
@@ -233,16 +228,15 @@ CMake
233228
-DARROW_JAVA_JNI_ENABLE_DEFAULT=ON \
234229
-DBUILD_TESTING=OFF \
235230
-DCMAKE_BUILD_TYPE=Release \
236-
-DCMAKE_INSTALL_LIBDIR=lib/<your system's architecture> \
237231
-DCMAKE_INSTALL_PREFIX=java-dist \
238232
-DCMAKE_PREFIX_PATH=$PWD/java-dist \
239233
-DProtobuf_ROOT=$PWD/../cpp-jni/protobuf_ep-install \
240234
-DProtobuf_USE_STATIC_LIBS=ON
241235
$ cmake --build java-jni --target install --config Release
242-
$ ls -latr java-dist/lib/<your system's architecture>/*_{jni,java}.*
243-
|__ libarrow_dataset_jni.dylib
244-
|__ libarrow_orc_jni.dylib
245-
|__ libgandiva_jni.dylib
236+
$ ls -latr java-dist/lib/
237+
|__ arrow_dataset_jni/
238+
|__ arrow_orc_jni/
239+
|__ gandiva_jni/
246240
247241
- To build all JNI libraries (Windows) except the JNI C Data Interface library:
248242

@@ -271,7 +265,6 @@ CMake
271265
-DARROW_WITH_ZLIB=ON ^
272266
-DARROW_WITH_ZSTD=ON ^
273267
-DCMAKE_BUILD_TYPE=Release ^
274-
-DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
275268
-DCMAKE_INSTALL_PREFIX=java-dist ^
276269
-DCMAKE_UNITY_BUILD=ON ^
277270
-GNinja
@@ -288,13 +281,12 @@ CMake
288281
-DARROW_JAVA_JNI_ENABLE_ORC=ON ^
289282
-DBUILD_TESTING=OFF ^
290283
-DCMAKE_BUILD_TYPE=Release ^
291-
-DCMAKE_INSTALL_LIBDIR=lib/x86_64 ^
292284
-DCMAKE_INSTALL_PREFIX=java-dist ^
293285
-DCMAKE_PREFIX_PATH=$PWD/java-dist
294286
$ cmake --build java-jni --target install --config Release
295287
$ dir "java-dist/bin"
296-
|__ arrow_orc_jni.dll
297-
|__ arrow_dataset_jni.dll
288+
|__ arrow_orc_jni/
289+
|__ arrow_dataset_jni/
298290
299291
Archery
300292
~~~~~~~
@@ -303,11 +295,11 @@ Archery
303295
304296
$ cd arrow
305297
$ archery docker run java-jni-manylinux-2014
306-
$ ls -latr java-dist/<your system's architecture>/
307-
|__ libarrow_cdata_jni.so
308-
|__ libarrow_dataset_jni.so
309-
|__ libarrow_orc_jni.so
310-
|__ libgandiva_jni.so
298+
$ ls -latr java-dist
299+
|__ arrow_cdata_jni/
300+
|__ arrow_dataset_jni/
301+
|__ arrow_orc_jni/
302+
|__ gandiva_jni/
311303
312304
Building Java JNI Modules
313305
-------------------------

0 commit comments

Comments
 (0)