[Native] Gradle-related tweaks to improve handling of the simdvec native library (#144539)

ldematte · web-flow · commit 568254df9ae8 · 2026-03-26T11:50:41.000+01:00
This PR introduces a couple of changes related to native libraries build;

    In the script used to build and publish the native vec library, force gradle to rebuild native intermediate files to ensure a clean and up-to-date binary. Under some conditions, we ended with object files that were not at the latest version. Probably these were extreme conditions (locally built libraries, experimenting different solutions), but better be safe. The library is tiny and takes seconds to build anyway.

    Update gradle build files of benchmarks and KnnIndexTester to explicitly depend on native-libraries. Most of the code in benchmarks and KnnIndexTester depends on the native simdvec library, and it happened in the past that we unknowingly run with older, not updated versions of the library. In particular, we want the :libs:native:native-libraries:extractLibs to appear in the task graph for both :benchmarks:run and :qa:vector:checkVec.
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
@@ -63,7 +63,7 @@ dependencies {
   }
   expression(project(path: ':modules:lang-expression', configuration: 'zip'))
   painless(project(path: ':modules:lang-painless', configuration: 'zip'))
-  nativeLib(project(':libs:native'))
+  nativeLib(project(path: ':libs:native:native-libraries', configuration: 'default'))
   api "org.openjdk.jmh:jmh-core:$versions.jmh"
   annotationProcessor "org.openjdk.jmh:jmh-generator-annprocess:$versions.jmh"
   // Dependencies of JMH
@@ -99,7 +99,7 @@ tasks.named("run").configure {
   dependsOn "copyExpression", "copyPainless", configurations.nativeLib
   systemProperty 'plugins.dir', "${buildDir}/plugins"
   systemProperty 'tests.index', "${buildDir}/index"
-  systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())
+  systemProperty 'es.nativelibs.path', providers.provider { TestUtil.getTestLibraryPath(configurations.nativeLib.asPath) }
 }
 
 tasks.named('test').configure {
diff --git a/libs/simdvec/native/Dockerfile.aarch64 b/libs/simdvec/native/Dockerfile.aarch64
@@ -4,7 +4,7 @@ RUN apt-get update
 RUN apt-get install -y gcc g++ openjdk-21-jdk
 COPY . /workspace
 WORKDIR /workspace
-RUN ./gradlew --quiet --console=plain clean buildSharedLibrary
+RUN ./gradlew --quiet --no-build-cache --console=plain clean buildSharedLibrary
 RUN strip --strip-unneeded build/output/libvec.so
 
 CMD ["cat", "build/output/libvec.so"]
diff --git a/libs/simdvec/native/Dockerfile.amd64 b/libs/simdvec/native/Dockerfile.amd64
@@ -4,7 +4,7 @@ RUN apt-get update
 RUN apt-get install -y gcc g++ openjdk-21-jdk
 COPY . /workspace
 WORKDIR /workspace
-RUN ./gradlew --quiet --console=plain clean buildSharedLibrary
+RUN ./gradlew --quiet --no-build-cache --console=plain clean buildSharedLibrary
 RUN strip --strip-unneeded build/output/libvec.so
 
 CMD ["cat", "build/output/libvec.so"]
diff --git a/libs/simdvec/native/publish_vec_binaries.sh b/libs/simdvec/native/publish_vec_binaries.sh
@@ -30,7 +30,7 @@ if curl -sS -I --fail --location "${ARTIFACTORY_REPOSITORY}/org/elasticsearch/ve
 fi
 
 echo 'Building Darwin binary...'
-./gradlew --quiet --console=plain clean vecAarch64SharedLibrary
+./gradlew --quiet --console=plain --no-build-cache clean vecAarch64SharedLibrary
 
 echo 'Building Linux binary...'
 mkdir -p build/libs/vec/shared/aarch64/
diff --git a/qa/vector/build.gradle b/qa/vector/build.gradle
@@ -14,6 +14,10 @@ import org.elasticsearch.gradle.VersionProperties
 apply plugin: 'elasticsearch.java'
 apply plugin: 'elasticsearch.build'
 
+configurations {
+  nativeLib
+}
+
 
 tasks.named("dependencyLicenses").configure {
   mapping from: /lucene-.*/, to: 'lucene'
@@ -86,6 +90,8 @@ dependencies {
     }
   }
 
+  nativeLib(project(path: ':libs:native:native-libraries', configuration: 'default'))
+
   testImplementation project(":libs:x-content")
   testImplementation project(":test:framework")
 }
@@ -205,7 +211,8 @@ tasks.register("checkVec", JavaExec) {
   // Configure logging to console
   systemProperty "es.logger.out", "console"
   systemProperty "es.logger.level", "INFO"  // Change to DEBUG if needed
-  systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../../libs/native/libraries/build/platform/").toString())
+  dependsOn configurations.nativeLib
+  systemProperty 'es.nativelibs.path', providers.provider { TestUtil.getTestLibraryPath(configurations.nativeLib.asPath) }
   jvmArgs '-Xms16g', '-Xmx16g', '-Djava.util.concurrent.ForkJoinPool.common.parallelism=8', '-XX:+UnlockDiagnosticVMOptions', '-XX:+DebugNonSafepoints', '-XX:+HeapDumpOnOutOfMemoryError'
   if (buildParams.getRuntimeJavaVersion().map { it.majorVersion.toInteger() }.get() >= 21) {
     jvmArgs '--add-modules=jdk.incubator.vector', '--enable-native-access=ALL-UNNAMED'
