-
Notifications
You must be signed in to change notification settings - Fork 582
Expand file tree
/
Copy pathbuild.sh
More file actions
executable file
·1111 lines (977 loc) · 37.6 KB
/
build.sh
File metadata and controls
executable file
·1111 lines (977 loc) · 37.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env bash
set -e
shopt -s extglob
#-----------------------------------------------------------------------------
# RediSearch Build Script
#
# This script handles building the RediSearch module and running tests.
# It supports various build configurations and test types.
#-----------------------------------------------------------------------------
# Get the absolute path to script directory
ROOT="$(cd "$(dirname "$0")" && pwd)"
BINROOT="$ROOT/bin"
#-----------------------------------------------------------------------------
# Default configuration values
#-----------------------------------------------------------------------------
COORD="oss" # Coordinator type: oss or rlec
DEBUG=0 # Debug build flag
PROFILE=0 # Profile build flag
FORCE=0 # Force clean build flag
VERBOSE=0 # Verbose output flag
QUICK=${QUICK:-0} # Quick test mode (subset of tests)
COV=${COV:-0} # Coverage mode (for building and testing)
BUILD_INTEL_SVS_OPT=${BUILD_INTEL_SVS_OPT:-0} # Use SVS pre-compiled library
# Enable Rust/C LTO. Requires Clang and lld (Linux only).
# Clang needs to have the same version as the LLVM version used by Rust.
# Check using `clang --version` and `rustc --version --verbose`.
LTO=0
# Test configuration (0=disabled, 1=enabled)
BUILD_TESTS=0 # Build test binaries
RUN_UNIT_TESTS=0 # Run C/C++ unit tests
RUN_RUST_TESTS=0 # Run Rust tests
RUN_RUST_VALGRIND=0 # Run Valgrind Rust tests
RUN_PYTEST=0 # Run Python tests
RUN_ALL_TESTS=0 # Run all test types
RUN_MICRO_BENCHMARKS=0 # Run micro-benchmarks
# Rust configuration
RUST_PROFILE="" # Which profile should be used to build/test Rust code
# If unspecified, the correct profile will be determined based
# the operations to be performed
RUN_MIRI=0 # Run Rust tests through miri to catch undefined behavior
RUST_DENY_WARNS=0 # Deny all Rust compiler warnings
RUST_TOOLCHAIN_MODIFIER="" # Rust toolchain to use (e.g., +nightly)
# Rust code is built first, so exclude benchmarking crates that link C code,
# since the static libraries they depend on haven't been built yet.
EXCLUDE_RUST_BENCHING_CRATES_LINKING_C="--exclude inverted_index_bencher --exclude rqe_iterators_bencher --exclude iterators_ffi"
# Retrieve our pinned nightly version.
NIGHTLY_VERSION=$(cat ${ROOT}/.rust-nightly)
#-----------------------------------------------------------------------------
# Function: parse_arguments
# Parse command-line arguments and set configuration variables
# Requires extglob to be enabled
#-----------------------------------------------------------------------------
parse_arguments() {
for arg in "$@"; do
# macOS only has bash 3.2 built-in, which doesn't support the more modern ${arg^^} syntax.
upper_arg=$(printf '%s' "$arg" | tr '[:lower:]' '[:upper:]')
case $upper_arg in
COORD=*)
COORD="${arg#*=}"
;;
DEBUG?(=1))
DEBUG=1
;;
PROFILE?(=1))
PROFILE=1
;;
TESTS?(=1))
BUILD_TESTS=1
;;
RUN_TESTS?(=1))
RUN_ALL_TESTS=1
;;
RUN_UNIT_TESTS?(=1))
RUN_UNIT_TESTS=1
;;
RUN_RUST_TESTS?(=1))
RUN_RUST_TESTS=1
;;
RUN_RUST_VALGRIND?(=1))
RUN_RUST_VALGRIND=1
;;
RUN_MICRO_BENCHMARKS?(=1))
RUN_MICRO_BENCHMARKS=1
;;
COV=*)
COV="${arg#*=}"
;;
RUN_PYTEST?(=1))
RUN_PYTEST=1
;;
EXT=*)
EXT="${arg#*=}"
;;
EXT_HOST=*)
if [[ "${arg#*=}" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
EXT_HOST="${arg#*=}"
else
echo "Invalid IP address: ${arg#*=}"
exit 1
fi
;;
EXT_PORT=*)
EXT_PORT="${arg#*=}"
;;
TEST=*)
TEST_FILTER="${arg#*=}"
;;
RUST_PROFILE=*)
RUST_PROFILE="${arg#*=}"
;;
RUST_DYN_CRT=*)
RUST_DYN_CRT="${arg#*=}"
;;
RUN_MIRI=*)
RUN_MIRI="${arg#*=}"
;;
RUST_DENY_WARNS=*)
RUST_DENY_WARNS="${arg#*=}"
;;
SAN=*)
SAN="${arg#*=}"
;;
FORCE?(=1))
FORCE=1
;;
VERBOSE?(=1))
VERBOSE=1
;;
QUICK=*)
QUICK="${arg#*=}"
;;
TEST_TIMEOUT=*)
TEST_TIMEOUT="${arg#*=}"
;;
SA=*)
SA="${arg#*=}"
;;
REDIS_STANDALONE=*)
REDIS_STANDALONE="${arg#*=}"
;;
BUILD_INTEL_SVS_OPT=*)
BUILD_INTEL_SVS_OPT="${arg#*=}"
;;
LTO?(=1))
LTO=1
;;
*)
# Pass all other arguments directly to CMake
CMAKE_ARGS="$CMAKE_ARGS -D${arg}"
;;
esac
done
}
#-----------------------------------------------------------------------------
# Function: setup_test_configuration
# Configure test settings based on input arguments
#-----------------------------------------------------------------------------
setup_test_configuration() {
# If any tests will be run, ensure BUILD_TESTS is enabled
if [[ "$RUN_ALL_TESTS" == "1" || "$RUN_UNIT_TESTS" == "1" || "$RUN_RUST_TESTS" == "1" || "$RUN_RUST_VALGRIND" == "1" || "$RUN_PYTEST" == "1" || "$RUN_MICRO_BENCHMARKS" == "1" ]]; then
if [[ "$BUILD_TESTS" != "1" ]]; then
echo "Test execution requested, enabling test build automatically"
BUILD_TESTS="1"
fi
fi
# If RUN_ALL_TESTS is enabled, enable all test types
if [[ "$RUN_ALL_TESTS" == "1" ]]; then
RUN_UNIT_TESTS=1
RUN_RUST_TESTS=1
RUN_PYTEST=1
fi
}
#-----------------------------------------------------------------------------
# Function: setup_build_environment
# Configure the build environment variables
#-----------------------------------------------------------------------------
setup_build_environment() {
# Determine Rust toolchain
if [[ -n "$SAN" || "$COV" == "1" || "$RUN_MIRI" == "1" ]]; then
# For coverage, we use the `nightly` compiler in order to include doc tests in the coverage computation.
# See https://github.com/taiki-e/cargo-llvm-cov/issues/2 for more details.
echo "Using nightly version: ${NIGHTLY_VERSION}"
RUST_TOOLCHAIN_MODIFIER="+$NIGHTLY_VERSION"
fi
# Determine build flavor
if [ "$SAN" == "address" ]; then
FLAVOR="debug-asan"
elif [[ "$RUN_MIRI" == "1" ]]; then
FLAVOR="debug-miri"
elif [[ "$DEBUG" == "1" ]]; then
FLAVOR="debug"
elif [[ "$COV" == "1" ]]; then
FLAVOR="debug-cov"
elif [[ "$PROFILE" == "1" ]]; then
FLAVOR="release-profile"
else
FLAVOR="release"
fi
# We must set the build target explicitly when running with a sanitizer to prevent the Rust flags from being applied to build
# scripts and procedural macros.
#
# See https://doc.rust-lang.org/beta/unstable-book/compiler-flags/sanitizer.html#build-scripts-and-procedural-macros
if [ "$SAN" == "address" ]; then
export CARGO_BUILD_TARGET="$(rustc $RUST_TOOLCHAIN_MODIFIER -vV | sed -n 's/host: //p')"
fi
# Disable ODR violation detection when building tests with ASAN. This is needed because both the
# shared redisearch.so and the test binaries link to the same static libraries, causing false
# positives (mostly in the Rust's compiler_builtins for `RSQRT_TAB`).
if [[ "$SAN" == "address" && "$BUILD_TESTS" == "1" ]]; then
export ASAN_OPTIONS=detect_odr_violation=0
fi
# Determine the correct Rust profile for both build and tests
# Only set RUST_PROFILE if it wasn't already set by the user
if [[ -z "$RUST_PROFILE" ]]; then
if [[ "$BUILD_TESTS" == "1" ]]; then
if [[ "$DEBUG" == "1" || -n "$SAN" || "$COV" == "1" || "$RUN_MIRI" == "1" ]]; then
RUST_PROFILE="dev"
else
if [[ "$RUN_MICRO_BENCHMARKS" == "1" ]]; then
# We don't want debug assertions to be enabled in microbenchmarks
RUST_PROFILE="release"
else
RUST_PROFILE="optimised_test"
fi
fi
else
if [[ "$DEBUG" == "1" ]]; then
RUST_PROFILE="dev"
else
RUST_PROFILE="release"
fi
fi
fi
# Get OS and architecture
OS_NAME=$(uname)
# Convert OS name to lowercase and convert Darwin to macos
if [[ "$OS_NAME" == "Darwin" ]]; then
OS_NAME="macos"
else
OS_NAME=$(echo "$OS_NAME" | tr '[:upper:]' '[:lower:]')
fi
# Get architecture and convert arm64 to aarch64
ARCH=$(uname -m)
if [[ "$ARCH" == "arm64" ]]; then
ARCH="aarch64"
elif [[ "$ARCH" == "x86_64" ]]; then
ARCH="x64"
fi
# Create full variant string for the build directory
FULL_VARIANT="${OS_NAME}-${ARCH}-${FLAVOR}"
# Set BINDIR based on configuration and FULL_VARIANT
if [[ "$COORD" == "oss" ]]; then
OUTDIR="search-community"
elif [[ "$COORD" == "rlec" ]]; then
OUTDIR="search-enterprise"
else
echo "COORD should be either oss or rlec"
exit 1
fi
# Set the final BINDIR using the full variant path
BINDIR="${BINROOT}/${FULL_VARIANT}/${OUTDIR}"
# Create compatibility symlink for aarch64 -> arm64v8 if needed
if [[ "$ARCH" == "aarch64" ]]; then
export ARM64V8_VARIANT="${OS_NAME}-arm64v8-${FLAVOR}"
export ARM64V8_BINROOT="${BINROOT}/${ARM64V8_VARIANT}"
fi
}
start_group() {
if [[ -n $GITHUB_ACTIONS ]]; then
echo "::group::$1"
fi
}
end_group() {
if [[ -n $GITHUB_ACTIONS ]]; then
echo "::endgroup::"
fi
}
#-----------------------------------------------------------------------------
# Function: prepare_coverage_capture
# Run lcov preparations before testing for coverage
#-----------------------------------------------------------------------------
prepare_coverage_capture() {
start_group "Code Coverage Preparation"
lcov --zerocounters --directory $BINROOT --base-directory $ROOT
lcov --capture --initial --directory $BINROOT --base-directory $ROOT -o $BINROOT/base.info
end_group
}
#-----------------------------------------------------------------------------
# Function: capture_coverage
# Capture coverage collected since `prepare_coverage_capture` was invoked
#-----------------------------------------------------------------------------
capture_coverage() {
NAME=${1:-cov} # Get output name. Defaults to `cov.info`
start_group "Code Coverage Capture ($NAME)"
# Capture coverage collected while running tests previously
lcov --capture --directory $BINROOT --base-directory $ROOT -o $BINROOT/test.info
# Accumulate results with the baseline captured before the test
lcov --add-tracefile $BINROOT/base.info --add-tracefile $BINROOT/test.info -o $BINROOT/full.info
# Extract only the coverage of the project source files
lcov --output-file $BINROOT/source.info --extract $BINROOT/full.info \
"$ROOT/src/*" \
"$ROOT/deps/thpool/*" \
# Remove coverage for directories we don't want (ignore if no file matches)
lcov -o $BINROOT/$NAME.info --ignore-errors unused --remove $BINROOT/source.info \
"*/tests/*" \
end_group
# Clean up temporary files
rm $BINROOT/base.info $BINROOT/test.info $BINROOT/full.info $BINROOT/source.info
}
#-----------------------------------------------------------------------------
# Function: prepare_cmake_arguments
# Prepare arguments to pass to CMake
#-----------------------------------------------------------------------------
prepare_cmake_arguments() {
# Initialize with base arguments
CMAKE_BASIC_ARGS="-DCOORD_TYPE=$COORD"
if [[ "$LTO" == "1" ]]; then
# LTO is only supported on Linux
if [[ "$OS_NAME" != "linux" ]]; then
echo "Error: LTO is only supported on Linux"
echo "Current OS: $OS_NAME"
exit 1
fi
# Enable Rust/C LTO by using clang and lld
# Determine compilers and linker:
# 1. Use CC/CXX/LD if set by user
# 2. Otherwise, try clang-$VERSION matching Rust's LLVM version
# 3. Otherwise, fall back to clang/clang++/lld
RUSTC_LLVM_VERSION=$(rustc --version --verbose | grep "LLVM version" | awk '{print $3}' | cut -d. -f1)
if [[ -z "$CC" ]]; then
if command -v "clang-$RUSTC_LLVM_VERSION" &>/dev/null; then
C_COMPILER="clang-$RUSTC_LLVM_VERSION"
else
C_COMPILER="clang"
fi
else
C_COMPILER="$CC"
if [[ ! "$C_COMPILER" =~ clang(-[0-9]+)?$ ]]; then
echo "Error: LTO requires clang as the C compiler"
echo "Current CC: $C_COMPILER"
echo "Please set CC to a clang-based compiler (e.g. clang, clang-nn)"
exit 1
fi
fi
if [[ -z "$CXX" ]]; then
if command -v "clang++-$RUSTC_LLVM_VERSION" &>/dev/null; then
CXX_COMPILER="clang++-$RUSTC_LLVM_VERSION"
else
CXX_COMPILER="clang++"
fi
else
CXX_COMPILER="$CXX"
if [[ ! "$CXX_COMPILER" =~ clang([+][+])?(-[0-9]+)?$ ]]; then
echo "Error: LTO requires clang++ as the C++ compiler"
echo "Current CXX: $CXX_COMPILER"
echo "Please set CXX to a clang-based compiler (e.g. clang++, clang++-nn)"
exit 1
fi
fi
if [[ -z "$LD" ]]; then
if command -v "lld-$RUSTC_LLVM_VERSION" &>/dev/null; then
LINKER="lld-$RUSTC_LLVM_VERSION"
else
LINKER="lld"
fi
else
LINKER="$LD"
if [[ ! "$LINKER" =~ lld(-[0-9]+)?$ ]]; then
echo "Error: LTO requires lld as the linker"
echo "Current LD: $LINKER"
echo "Please set LD to lld (e.g. lld, lld-nn)"
exit 1
fi
fi
# Check LLVM version compatibility between rustc and clang
# Use 'sed -E' for compatibility with both GNU sed and BSD sed
CLANG_LLVM_VERSION=$($C_COMPILER --version | head -n1 | sed -En 's/.*version ([0-9]+).*/\1/p' | head -n1)
if [[ -z "$RUSTC_LLVM_VERSION" || -z "$CLANG_LLVM_VERSION" ]]; then
echo "Error: Could not detect LLVM versions for rustc and clang."
echo "Cross-language LTO requires matching LLVM major versions."
echo "Rust LLVM version: $RUSTC_LLVM_VERSION"
echo "Clang LLVM version: $CLANG_LLVM_VERSION"
exit 1
fi
if [[ "$RUSTC_LLVM_VERSION" != "$CLANG_LLVM_VERSION" ]]; then
echo "Error: LLVM version mismatch between rustc and clang"
echo "Rust uses LLVM $RUSTC_LLVM_VERSION (from: rustc --version --verbose)"
echo "Clang uses LLVM $CLANG_LLVM_VERSION (from: $C_COMPILER --version)"
echo ""
echo "Cross-language LTO requires matching LLVM major versions."
echo "Please either:"
echo " 1. Install clang-$RUSTC_LLVM_VERSION"
echo " 2. Or build without LTO by removing the 'LTO' argument"
exit 1
fi
echo "Enabling C/Rust LTO"
# LLVM version alignment with the Rust compiler forces us to build with
# a rather recent version of clang (>=21.x.y).
# This can cause issues on older Linux distributions: if we're not careful,
# the .so we produce may rely on C++ symbols that don't exist in the
# C++ header files available at runtime (i.e. the ones provided by the
# system-level `gcc`/`g++` toolchain).
# To prevent this compile-time/runtime header mismatch, we force clang
# to use the C++ headers provided by the system's `g++` installation.
# This requires us to combine a few different flags and guardrails:
# * `--gcc-install-dir`, to point `clang` at artefacts (crtbegin.o, libgcc, etc.)
# for a _specific_ version of `gcc`
# * `-nostdinc++`, to disable standard `#include` directives for the C++
# standard library
# * `-isystem <dir>`, to control what paths are included in search space for
# C++ standard headers
# * An after-the-fact check to ensure we haven't included unwanted headers in
# the search
GCC_COMMON_FLAGS=""
GCC_CXX_FLAGS=""
if command -v g++ &>/dev/null; then
# Extract the C++ include paths that system g++ actually uses.
_cxx_includes=$(g++ -E -x c++ -v /dev/null 2>&1 | \
sed -n '/#include <\.\.\.>/,/^End/{ /^ /p }' | \
sed 's/^ *//' | \
grep -E '(/c\+\+/|/backward)') || true
# Point clang at g++'s include paths, disabling its default `#include`s
if [[ -n "$_cxx_includes" ]]; then
GCC_CXX_FLAGS="-nostdinc++"
while IFS= read -r dir; do
GCC_CXX_FLAGS+=" -isystem ${dir}"
done <<< "$_cxx_includes"
else
echo "Error: failed to extract C++ include paths from the system g++" >&2
exit 1
fi
# Pin `gcc`'s internal library dir (for crtbegin.o, libgcc, etc.)
GCC_INSTALL_DIR=$(gcc -print-search-dirs | sed -n 's/^install: //p')
if [[ -n "$GCC_INSTALL_DIR" ]]; then
GCC_COMMON_FLAGS="--gcc-install-dir=${GCC_INSTALL_DIR}"
fi
# --- Diagnostic: verify C++ header pinning ---
echo ">>> GCC common flags: ${GCC_COMMON_FLAGS}"
echo ">>> GCC C++ flags: ${GCC_CXX_FLAGS}"
echo ">>> Installed C++ header directories:"
ls -d /usr/include/c++/*/ 2>/dev/null || echo " (none found)"
echo ">>> Clang C++ include search paths:"
_search_paths=$($CXX_COMPILER ${GCC_COMMON_FLAGS} ${GCC_CXX_FLAGS} -x c++ -v -fsyntax-only /dev/null 2>&1 \
| sed -n '/#include <\.\.\.>/,/^End/p')
echo "$_search_paths"
# Fail if clang's actual search paths include C++ headers from a GCC other than system
_sys_gcc_major=$(gcc -dumpversion | cut -d. -f1)
_bad_paths=$(echo "$_search_paths" | grep -E "/c\+\+/[0-9]+" | grep -vE "/c\+\+/${_sys_gcc_major}(\.[0-9]+){0,2}(/|$)" || true)
if [[ -n "$_bad_paths" ]]; then
echo "ERROR: Clang sees C++ headers from a GCC version other than system GCC ${_sys_gcc_major}:"
echo "$_bad_paths"
echo " C++ header pinning is not working correctly."
echo " This will cause GLIBCXX symbol mismatch at link time."
exit 1
fi
fi
# Pass LTO C/CXX flags to CMake via CFLAGS/CXXFLAGS env vars so CMake picks them
# up without word-splitting issues.
# Note: we assume there are no spaces in system C++ include paths.
export CFLAGS="${CFLAGS:+${CFLAGS} }${GCC_COMMON_FLAGS}"
export CXXFLAGS="${CXXFLAGS:+${CXXFLAGS} }${GCC_COMMON_FLAGS}${GCC_CXX_FLAGS:+ ${GCC_CXX_FLAGS}}"
# Export CC/CXX so that Rust's cc crate also uses clang, matching the
# clang-specific flags in CFLAGS/CXXFLAGS (e.g. --gcc-install-dir).
export CC="$C_COMPILER"
export CXX="$CXX_COMPILER"
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS \
-DCMAKE_C_COMPILER=$C_COMPILER \
-DCMAKE_CXX_COMPILER=$CXX_COMPILER \
-DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=$LINKER \
-DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=$LINKER \
-DCMAKE_MODULE_LINKER_FLAGS=-fuse-ld=$LINKER \
-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=true"
# Include LLVM bitcode information for cross-language LTO
RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-C linker-plugin-lto -C linker=$C_COMPILER -C link-arg=-fuse-ld=$LINKER"
if [[ -n "$GCC_INSTALL_DIR" ]]; then
RUSTFLAGS="$RUSTFLAGS -C link-arg=--gcc-install-dir=${GCC_INSTALL_DIR}"
fi
fi
if [[ "$BUILD_TESTS" == "1" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DBUILD_SEARCH_UNIT_TESTS=ON"
fi
if [[ -n "$SAN" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DSAN=$SAN"
DEBUG="1"
fi
if [[ "$COV" == "1" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCOV=1"
DEBUG=1
fi
if [[ "$PROFILE" != 0 ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DPROFILE=$PROFILE"
# We shouldn't run profile with debug - so we fail the build
if [[ "$DEBUG" == "1" ]]; then
echo "Error: Cannot run profile with debug/sanitizer/coverage"
exit 1
fi
fi
# Set build type
if [[ "$DEBUG" == "1" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_BUILD_TYPE=Debug"
else
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_BUILD_TYPE=RelWithDebInfo"
fi
# Ensure output file is always .so even on macOS
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_SHARED_LIBRARY_SUFFIX=.so"
# Enable sccache for C/C++ compilation caching if available.
# Prefer SCCACHE_PATH (set by sccache-action in CI with the full path), otherwise look on PATH.
SCCACHE="${SCCACHE_PATH:-$(command -v sccache 2>/dev/null || true)}"
if [[ -n "$SCCACHE" && -x "$SCCACHE" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_C_COMPILER_LAUNCHER=$SCCACHE -DCMAKE_CXX_COMPILER_LAUNCHER=$SCCACHE"
echo "Using sccache for C/C++ compilation caching"
fi
# Add caching flags to prevent using old configurations
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -UCMAKE_TOOLCHAIN_FILE"
if [[ "$OS_NAME" == "macos" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
fi
if [[ "$BUILD_INTEL_SVS_OPT" == "yes" || "$BUILD_INTEL_SVS_OPT" == "1" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DBUILD_INTEL_SVS_OPT=ON"
else
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DSVS_SHARED_LIB=OFF"
fi
# Handle RUST_DYN_CRT flag for Alpine Linux compatibility
if [[ "$RUST_DYN_CRT" == "1" ]]; then
# Add the dynamic C runtime flag to RUSTFLAGS
RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-C target-feature=-crt-static"
fi
# MOD-14916: inline LSE atomics for Rust on AArch64. `-C target-cpu=neoverse-n1`
# implies +lse so rustc emits LDADDH/LDADD instead of an ldxrh/stxrh LL/SC loop.
# macOS is excluded to match the CMake NOT APPLE gate: Apple Silicon's default
# target-cpu (apple-m1) is already ≥Armv8.5-a with inline LSE, so overriding it
# with neoverse-n1 would only downgrade scheduling.
if [[ "$ARCH" == "aarch64" && "$OS_NAME" != "macos" ]]; then
RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-C target-cpu=neoverse-n1"
fi
# Set up RUSTFLAGS for warnings
if [[ "$RUST_DENY_WARNS" == "1" ]]; then
RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-D warnings"
fi
# Ensure we can compute coverage across the FFI boundary
if [[ $OS_NAME != "macos" && $COV == "1" ]]; then
# Needs the C code to link on gcov
RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} } -C link-args=-lgcov"
# Doctests are compiled by rustdoc, which uses RUSTDOCFLAGS rather than
# RUSTFLAGS for its link step. Without this, doctests that pull in
# gcov-instrumented C objects (via transitive deps on FFI crates) fail
# to link with undefined `__gcov_*` symbols.
RUSTDOCFLAGS="${RUSTDOCFLAGS:+${RUSTDOCFLAGS} }-C link-args=-lgcov"
export RUSTDOCFLAGS
fi
if [[ $SAN == "address" ]]; then
# Add ASAN flags to RUSTFLAGS (following RedisJSON pattern)
# -Zsanitizer=address enables ASAN in Rust
RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-Zsanitizer=address"
fi
# Workaround for macOS 14:
# Apple's ld (through Apple clang 16 / Xcode 16.2) has an ARM64 bug that
# misaligns symbols, causing "not 8-byte aligned" LDR/STR errors. Fixed in
# Apple clang 17 (Xcode 16.4+). Use LLVM's lld as a workaround when needed.
if [[ "$OS_NAME" == "macos" ]]; then
APPLE_CLANG_MAJOR=$(cc --version 2>/dev/null | head -1 | grep -oE 'version [0-9]+' | grep -oE '[0-9]+')
if [[ -n "$APPLE_CLANG_MAJOR" && "$APPLE_CLANG_MAJOR" -lt 17 ]]; then
# llvm@17 provides ld64.lld; the project's llvm@21 doesn't include lld.
local lld_path="$(brew --prefix)/opt/llvm@17/bin/ld64.lld"
if [[ -x "$lld_path" ]]; then
echo "Apple clang $APPLE_CLANG_MAJOR < 17: using llvm@17's ld64.lld to work around ARM64 alignment bug"
RUSTFLAGS="${RUSTFLAGS:+${RUSTFLAGS} }-C link-arg=-fuse-ld=${lld_path}"
else
echo "WARNING: Apple clang $APPLE_CLANG_MAJOR has a known ARM64 linker bug but ld64.lld is not installed at ${lld_path}"
fi
fi
fi
# Export RUSTFLAGS so it's available to the Rust build process
export RUSTFLAGS
# RUSTFLAGS will be passed as environment variable to avoid quoting issues
# This prevents CMake argument parsing from truncating complex flag values
if [[ "$RUST_PROFILE" != "" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DRUST_PROFILE=$RUST_PROFILE"
fi
if [[ -n "$CARGO_BUILD_TARGET" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DCARGO_BUILD_TARGET=$CARGO_BUILD_TARGET"
fi
if [[ -n "$RUST_TOOLCHAIN_MODIFIER" ]]; then
CMAKE_BASIC_ARGS="$CMAKE_BASIC_ARGS -DRUST_TOOLCHAIN_MODIFIER=$RUST_TOOLCHAIN_MODIFIER"
fi
}
#-----------------------------------------------------------------------------
# Function: run_cmake
# Run CMake to configure the build
#-----------------------------------------------------------------------------
run_cmake() {
# Create build directory and ensure any parent directories exist
mkdir -p "$BINDIR"
cd "$BINDIR"
# Create compatibility symlink for aarch64 -> arm64v8 if needed
if [[ "$ARCH" == "aarch64" && -n "$ARM64V8_BINROOT" ]]; then
if [[ ! -e "$ARM64V8_BINROOT" ]]; then
echo "Creating compatibility symlink: $ARM64V8_BINROOT -> ${BINROOT}/${FULL_VARIANT}"
ln -sf "${FULL_VARIANT}" "$ARM64V8_BINROOT"
fi
fi
# Clean up any cached CMake configuration if force is enabled
if [[ "$FORCE" == "1" ]]; then
echo "Cleaning CMake cache..."
rm -f CMakeCache.txt
rm -rf CMakeFiles
fi
echo "Configuring CMake..."
echo "Build directory: $BINDIR"
# Run CMake with all the flags
if [[ "$FORCE" == "1" || ! -f "$BINDIR/Makefile" ]]; then
CMAKE_CMD="cmake $ROOT $CMAKE_BASIC_ARGS $CMAKE_ARGS"
echo "$CMAKE_CMD"
# If verbose, dump all CMake variables before and after configuration
if [[ "$VERBOSE" == "1" ]]; then
echo "Running CMake with verbose output..."
RUSTFLAGS="$RUSTFLAGS" $CMAKE_CMD --trace-expand
else
RUSTFLAGS="$RUSTFLAGS" $CMAKE_CMD
fi
fi
}
#-----------------------------------------------------------------------------
# Function: build_project
# Build the RediSearch project using Make
#-----------------------------------------------------------------------------
build_project() {
# redisearch_rs is now built automatically by CMake
# Determine number of parallel jobs for make
if command -v nproc &> /dev/null; then
NPROC=$(nproc)
elif command -v sysctl &> /dev/null && [[ "$OS_NAME" == "macos" ]]; then
NPROC=$(sysctl -n hw.physicalcpu)
else
NPROC=4 # Default if we can't determine
fi
echo "Building RediSearch with $NPROC parallel jobs..."
make -j "$NPROC"
# Build test dependencies if needed
build_test_dependencies
# Report build success
echo "Build complete. Artifacts in $BINDIR"
}
#-----------------------------------------------------------------------------
# Function: build_test_dependencies
# Build additional dependencies needed for tests
#-----------------------------------------------------------------------------
build_test_dependencies() {
if [[ "$BUILD_TESTS" == "1" ]]; then
# Ensure ext-example binary gets compiled
if [[ -d "$ROOT/tests/ctests/ext-example" ]]; then
echo "Building ext-example for unit tests..."
# Check if we're already in the build directory
if [[ "$PWD" != "$BINDIR" ]]; then
cd "$BINDIR"
fi
# The example_extension target is created by CMake in the build directory
# First check if the target exists in this build
if grep -q "example_extension" Makefile 2>/dev/null || (make -q example_extension 2>/dev/null); then
make example_extension
else
# If the target doesn't exist, we need to ensure the test was properly configured
echo "Warning: 'example_extension' target not found in Makefile"
echo "Checking for extension binary..."
# Check if extension was already built by a previous run
EXTENSION_PATH="$BINDIR/example_extension/libexample_extension.so"
if [[ -f "$EXTENSION_PATH" ]]; then
echo "Extension binary already exists at: $EXTENSION_PATH"
else
echo "Extension binary not found. Some tests may fail."
echo "Try running 'make example_extension' manually in $BINDIR"
fi
fi
# Export extension path for tests
EXTENSION_PATH="$BINDIR/example_extension/libexample_extension.so"
if [[ -f "$EXTENSION_PATH" ]]; then
echo "Example extension located at: $EXTENSION_PATH"
export EXT_TEST_PATH="$EXTENSION_PATH"
else
echo "Warning: Could not find example extension at $EXTENSION_PATH"
echo "Some tests may fail if they depend on this extension"
fi
fi
fi
}
#-----------------------------------------------------------------------------
# Function: run_unit_tests
# Run C/C++ unit tests
#-----------------------------------------------------------------------------
run_unit_tests() {
if [[ "$RUN_UNIT_TESTS" != "1" ]]; then
return 0
fi
echo "Running unit tests..."
# Set test environment variables if needed
if [[ "$OS_NAME" == "macos" ]]; then
echo "Running unit tests on macOS"
# On macOS, we may need to set DYLD_LIBRARY_PATH or similar
# Uncomment if needed:
# export DYLD_LIBRARY_PATH="$BINDIR:$DYLD_LIBRARY_PATH"
fi
# Set up environment variables for the unit-tests script
export BINDIR
# Set up test filter if provided
if [[ -n "$TEST_FILTER" ]]; then
echo "Running tests matching: $TEST_FILTER"
export TEST="$TEST_FILTER"
fi
if [[ $COV == 1 ]]; then
prepare_coverage_capture
fi
# Set verbose mode if requested
if [[ "$VERBOSE" == "1" ]]; then
export VERBOSE=1
fi
# Set sanitizer mode if requested
if [[ "$SAN" == "address" ]]; then
export SAN="address"
fi
# Call the unit-tests script from the sbin directory
echo "Calling $ROOT/sbin/unit-tests"
"$ROOT/sbin/unit-tests"
# Check test results
UNIT_TEST_RESULT=$?
if [[ $UNIT_TEST_RESULT -eq 0 ]]; then
echo "All unit tests passed!"
if [[ $COV == 1 ]]; then
capture_coverage unit
fi
else
echo "Some unit tests failed. Check the test logs above for details."
HAS_FAILURES=1
fi
}
#-----------------------------------------------------------------------------
# Function: run_rust_tests
# Run Rust tests
#-----------------------------------------------------------------------------
run_rust_tests() {
if [[ "$RUN_RUST_TESTS" != "1" ]]; then
return 0
fi
echo "Running Rust tests..."
# Tell Rust build scripts where to find the compiled static libraries
export BINDIR
# Set Rust test environment
RUST_DIR="$ROOT/src/redisearch_rs"
CARGO_BUILD_FLAGS=""
# Add Rust test extensions
if [[ $COV == 1 ]]; then
RUST_TEST_COMMAND="llvm-cov test"
# We exclude Rust benchmarking crates that link to C code when computing coverage.
# On one side, we aren't interested in coverage of those utilities.
# On top of that, it causes linking issues since, when computing coverage, it seems to
# require C symbols to be defined even if they aren't invoked at runtime.
RUST_TEST_OPTIONS="
--profile=$RUST_PROFILE
--doctests
$EXCLUDE_RUST_BENCHING_CRATES_LINKING_C
--codecov
--ignore-filename-regex="varint_bencher/*,trie_bencher/*,inverted_index_bencher/*"
--output-path=$BINROOT/rust_cov.info
"
elif [[ "$RUN_MIRI" == "1" ]]; then
RUST_TEST_COMMAND="miri nextest run"
RUST_TEST_OPTIONS="--cargo-profile=$RUST_PROFILE"
elif [[ "$SAN" == "address" ]]; then
# We must rebuild the Rust standard library to get sanitizer coverage
# for its functions.
# Since --build-std is a cargo flag (not rustc), we set it separately
CARGO_BUILD_FLAGS="${CARGO_BUILD_FLAGS:+${CARGO_BUILD_FLAGS} }-Zbuild-std"
RUST_TEST_COMMAND="nextest run"
# The doc tests are disabled under ASAN to avoid issues with linking to the sanitizer runtime
# in doc tests.
RUST_TEST_OPTIONS="--tests --cargo-profile=$RUST_PROFILE $EXCLUDE_RUST_BENCHING_CRATES_LINKING_C"
else
RUST_TEST_COMMAND="nextest run"
RUST_TEST_OPTIONS="--cargo-profile=$RUST_PROFILE"
fi
# Run cargo test with the appropriate filter
cd "$RUST_DIR"
RUSTFLAGS="${RUSTFLAGS}" cargo $RUST_TOOLCHAIN_MODIFIER $CARGO_BUILD_FLAGS $RUST_TEST_COMMAND $RUST_TEST_OPTIONS --workspace $TEST_FILTER
# Check test results
RUST_TEST_RESULT=$?
if [[ $RUST_TEST_RESULT -eq 0 ]]; then
echo "All Rust tests passed!"
else
echo "Some Rust tests failed. Check the test logs above for details."
HAS_FAILURES=1
fi
}
#-----------------------------------------------------------------------------
# Function: run_rust_valgrind_tests
# Run Rust Valgrind tests (to detect memory leaks)
#-----------------------------------------------------------------------------
run_rust_valgrind_tests() {
if [[ "$RUN_RUST_VALGRIND" != "1" ]]; then
return 0
fi
echo "Running Rust tests..."
# Set Rust test environment
RUST_DIR="$ROOT/src/redisearch_rs"
cd "$RUST_DIR"
if [[ "$OS_NAME" == "macos" ]]; then
# no valgrind on apple silicon... so...
echo "The valgrind test target is only supported on Linux"
HAS_FAILURES=1
return 0
else
# Run cargo valgrind with the appropriate filter
VALGRINDFLAGS=--suppressions=$PWD/valgrind.supp \
RUSTFLAGS="${RUSTFLAGS}" \
cargo valgrind test \
--profile=$RUST_PROFILE \
--workspace $TEST_FILTER \
-- --nocapture
fi
# Check test results
RUST_TEST_RESULT=$?
if [[ $RUST_TEST_RESULT -eq 0 ]]; then
echo "Rust Valgrind test passed!"
else
echo "Some Rust valgrind tests failed. Check the test logs above for details."
HAS_FAILURES=1
fi
}
#-----------------------------------------------------------------------------
# Function: run_python_tests
# Run Python behavioral tests
#-----------------------------------------------------------------------------
run_python_tests() {
if [[ "$RUN_PYTEST" != "1" ]]; then
return 0
fi
echo "Running Python behavioral tests..."
# Locate the built module
MODULE_PATH="$BINDIR/redisearch.so"
if [[ ! -f "$MODULE_PATH" && -f "$BINDIR/module-enterprise.so" ]]; then
MODULE_PATH="$BINDIR/module-enterprise.so"
fi
if [[ ! -f "$MODULE_PATH" ]]; then
echo "Error: Cannot find RediSearch module binary in $BINDIR"
exit 1
fi
# Set up environment variables required by runtests.sh
export MODULE="$(realpath "$MODULE_PATH")"
export BINROOT
export FULL_VARIANT
export BINDIR
export REJSON="${REJSON:-1}"
export REJSON_BRANCH="${REJSON_BRANCH:-master}"
export REJSON_PATH
export REJSON_ARGS
export TEST
export FORCE
export PARALLEL="${PARALLEL:-1}"
export LOG_LEVEL="${LOG_LEVEL:-debug}"
export TEST_TIMEOUT
export REDIS_STANDALONE="${REDIS_STANDALONE:-1}"
export SA="${SA:-$REDIS_STANDALONE}"
export COV
export EXT=${EXT-"RUN"}
export EXT_HOST=${EXT_HOST-"127.0.0.1"}
export EXT_PORT=${EXT_PORT-6379}
# Set up test filter if provided
if [[ -n "$TEST_FILTER" ]]; then
export TEST="$TEST_FILTER"
echo "Running Python tests matching: $TEST_FILTER"
fi
# Enable quick mode if requested (run only a subset of tests)
if [[ "$QUICK" == "1" ]]; then
echo "Running in QUICK mode - using a subset of tests"
export QUICK=1
fi
# Enable verbose mode if requested
if [[ "$VERBOSE" == "1" ]]; then
export VERBOSE=1
export RLTEST_VERBOSE=1
fi
if [[ $COV == 1 ]]; then
prepare_coverage_capture
fi
# Use the runtests.sh script for Python tests
TESTS_SCRIPT="$ROOT/tests/pytests/runtests.sh"
echo "Running Python tests with module at: $MODULE"
# Run the tests from the ROOT directory with the requested params
cd "$ROOT"