Skip to content

Commit 95a93e5

Browse files
Dead2Dead2
committed
Speed up benchmarks when run as part of gtest as it does not check data
for correctness, making it only run each benchmark for 1 iteration, instead of thousands or hundreds of thousands. Add a separate CI step to crashtest benchmarks without collecting any coverage data. Activate benchmarks in more arches. Disable some warnings to avoid errors in compiling google benchmark. Remove separate benchmark CI job, now included in other jobs instead.
1 parent b2a0ffb commit 95a93e5

3 files changed

Lines changed: 30 additions & 20 deletions

File tree

‎.github/workflows/cmake.yml‎

Lines changed: 28 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,17 @@ jobs:
1818
- name: Ubuntu 20.04 GCC
1919
os: ubuntu-20.04
2020
compiler: gcc
21+
cmake-args: -DWITH_BENCHMARKS=ON
2122
cxx-compiler: g++
2223

2324
- name: Ubuntu GCC ASAN
2425
os: ubuntu-latest
2526
compiler: gcc
2627
cxx-compiler: g++
27-
cmake-args: -DWITH_SANITIZER=Address
28+
cxxflags: -Wno-maybe-uninitialized
29+
cmake-args: -DWITH_SANITIZER=Address -DWITH_BENCHMARKS=ON
2830
codecov: ubuntu_gcc
2931

30-
- name: Ubuntu GCC Benchmark
31-
os: ubuntu-latest
32-
compiler: gcc
33-
cxx-compiler: g++
34-
cmake-args: -DWITH_BENCHMARKS=ON
35-
codecov: ubuntu_gcc_benchmark
36-
3732
- name: Ubuntu GCC Native Instructions
3833
os: ubuntu-latest
3934
compiler: gcc
@@ -93,7 +88,7 @@ jobs:
9388
os: ubuntu-latest
9489
compiler: gcc
9590
cxx-compiler: g++
96-
cmake-args: -DCMAKE_C_FLAGS=-m32 -DCMAKE_CXX_FLAGS=-m32 -DCMAKE_SHARED_LINKER_FLAGS=-m32 -DCMAKE_EXE_LINKER_FLAGS=-m32
91+
cmake-args: -DCMAKE_C_FLAGS=-m32 -DCMAKE_CXX_FLAGS=-m32 -DCMAKE_SHARED_LINKER_FLAGS=-m32 -DCMAKE_EXE_LINKER_FLAGS=-m32 -DWITH_BENCHMARKS=ON
9792
packages: gcc-multilib g++-multilib
9893
codecov: ubuntu_gcc_m32
9994

@@ -171,7 +166,8 @@ jobs:
171166

172167
- name: Ubuntu GCC ARM HF ASAN
173168
os: ubuntu-latest
174-
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-armhf.cmake -DWITH_SANITIZER=Address
169+
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-armhf.cmake -DWITH_SANITIZER=Address -DWITH_BENCHMARKS=ON
170+
cxxflags: -Wno-psabi -Wno-maybe-uninitialized
175171
asan-options: detect_leaks=0
176172
packages: qemu-user crossbuild-essential-armhf
177173
gcov-exec: arm-linux-gnueabihf-gcov
@@ -202,7 +198,7 @@ jobs:
202198

203199
- name: Ubuntu GCC AARCH64 ASAN
204200
os: ubuntu-22.04
205-
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-aarch64.cmake -DWITH_SANITIZER=Address
201+
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-aarch64.cmake -DWITH_SANITIZER=Address -DWITH_BENCHMARKS=ON
206202
asan-options: detect_leaks=0
207203
packages: qemu-user gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libc-dev-arm64-cross
208204
gcov-exec: aarch64-linux-gnu-gcov
@@ -238,7 +234,7 @@ jobs:
238234

239235
- name: Ubuntu GCC MIPS64
240236
os: ubuntu-latest
241-
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-mips64.cmake
237+
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-mips64.cmake -DWITH_BENCHMARKS=ON
242238
packages: qemu-user crossbuild-essential-mips64
243239
gcov-exec: mips64-linux-gnuabi64-gcov
244240
codecov: ubuntu_gcc_mips64
@@ -259,7 +255,7 @@ jobs:
259255

260256
- name: Ubuntu GCC PPC64
261257
os: ubuntu-latest
262-
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-powerpc64.cmake
258+
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-powerpc64.cmake -DWITH_BENCHMARKS=ON
263259
packages: qemu-user gcc-powerpc64-linux-gnu g++-powerpc64-linux-gnu libc-dev-ppc64-cross
264260
ldflags: -static
265261
gcov-exec: powerpc64-linux-gnu-gcov
@@ -307,7 +303,7 @@ jobs:
307303

308304
- name: Ubuntu GCC RISC-V
309305
os: ubuntu-latest
310-
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-riscv.cmake
306+
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-riscv.cmake -DWITH_BENCHMARKS=ON
311307
packages: qemu-user crossbuild-essential-riscv64
312308
gcov-exec: riscv64-linux-gnu-gcov
313309
codecov: ubuntu_gcc_riscv64
@@ -328,7 +324,7 @@ jobs:
328324

329325
- name: Ubuntu GCC S390X ASAN
330326
os: ubuntu-latest
331-
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-s390x.cmake -DWITH_SANITIZER=Address
327+
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-s390x.cmake -DWITH_SANITIZER=Address -DWITH_BENCHMARKS=ON
332328
asan-options: detect_leaks=0
333329
packages: qemu-user crossbuild-essential-s390x
334330
ldflags: -static
@@ -350,7 +346,7 @@ jobs:
350346
cxx-compiler: g++
351347
cmake-args: >-
352348
${{ github.repository != 'zlib-ng/zlib-ng' && '-DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-s390x.cmake' || '' }}
353-
-DWITH_DFLTCC_DEFLATE=ON -DWITH_DFLTCC_INFLATE=ON -DWITH_SANITIZER=Address
349+
-DWITH_DFLTCC_DEFLATE=ON -DWITH_DFLTCC_INFLATE=ON -DWITH_SANITIZER=Address -DWITH_BENCHMARKS=ON
354350
packages: qemu-user gcc-s390x-linux-gnu g++-s390x-linux-gnu libc-dev-s390x-cross
355351
asan-options: detect_leaks=0
356352
ldflags: -static
@@ -397,8 +393,9 @@ jobs:
397393

398394
- name: Ubuntu MinGW x86_64
399395
os: ubuntu-latest
400-
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-mingw-x86_64.cmake
396+
cmake-args: -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain-mingw-x86_64.cmake -DWITH_BENCHMARKS=ON
401397
packages: wine wine64 gcc-mingw-w64 g++-mingw-w64
398+
cxxflags: -Wno-unused-parameter
402399
ldflags: -static
403400
gcov-exec: x86_64-w64-mingw32-gcov-posix
404401
codecov: ubuntu_gcc_mingw_x86_64
@@ -588,7 +585,7 @@ jobs:
588585
os: macos-13
589586
compiler: clang
590587
cxx-compiler: clang++
591-
cmake-args: -DCMAKE_OSX_DEPLOYMENT_TARGET=10.10
588+
cmake-args: -DCMAKE_OSX_DEPLOYMENT_TARGET=10.10 -DWITH_BENCHMARKS=ON
592589
ldflags: -ld_classic
593590

594591
- name: macOS Clang ASAN
@@ -727,6 +724,7 @@ jobs:
727724
CC: ${{ matrix.compiler }}
728725
CXX: ${{ matrix.cxx-compiler }}
729726
CFLAGS: ${{ matrix.cflags }}
727+
CXXFLAGS: ${{ matrix.cxxflags }}
730728
LDFLAGS: ${{ matrix.ldflags }}
731729
CI: true
732730

@@ -736,7 +734,7 @@ jobs:
736734
- name: Run test cases
737735
# Don't run tests on Windows ARM
738736
if: runner.os != 'Windows' || contains(matrix.name, 'ARM') == false
739-
run: ctest --verbose -C Release --output-on-failure --max-width 120 -j ${{ matrix.parallels-jobs || '3' }}
737+
run: ctest --verbose -C Release -E benchmark_zlib --output-on-failure --max-width 120 -j ${{ matrix.parallels-jobs || '3' }}
740738
working-directory: ${{ matrix.build-dir || '.' }}
741739
env:
742740
ASAN_OPTIONS: ${{ matrix.asan-options || 'verbosity=0' }}:abort_on_error=1:halt_on_error=1
@@ -771,6 +769,17 @@ jobs:
771769
path: ${{ matrix.codecov }}.xml
772770
retention-days: 1
773771

772+
- name: Test benchmarks (crashtest only, no coverage data collection)
773+
if: contains(matrix.cmake-args, '-DWITH_BENCHMARKS=ON')
774+
run: ctest --verbose -C Release -R ^benchmark_zlib$ --output-on-failure --max-width 120 -j ${{ matrix.parallels-jobs || '3' }}
775+
working-directory: ${{ matrix.build-dir || '.' }}
776+
env:
777+
ASAN_OPTIONS: ${{ matrix.asan-options || 'verbosity=0' }}:abort_on_error=1:halt_on_error=1
778+
MSAN_OPTIONS: ${{ matrix.msan-options || 'verbosity=0' }}:abort_on_error=1:halt_on_error=1
779+
TSAN_OPTIONS: ${{ matrix.tsan-options || 'verbosity=0' }}:abort_on_error=1:halt_on_error=1
780+
LSAN_OPTIONS: ${{ matrix.lsan-options || 'verbosity=0' }}:abort_on_error=1:halt_on_error=1
781+
UBSAN_OPTIONS: ${{ matrix.ubsan-options || 'verbosity=0' }}:print_stacktrace=1:abort_on_error=1:halt_on_error=1
782+
774783
- name: Upload build errors
775784
uses: actions/upload-artifact@v4
776785
if: failure()

‎test/benchmarks/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ if(WIN32)
6363
endif()
6464

6565
add_test(NAME benchmark_zlib
66-
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:benchmark_zlib>)
66+
COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:benchmark_zlib> "--benchmark_min_time=0")
6767

6868
if(WITH_BENCHMARK_APPS)
6969
option(BUILD_ALT_BENCH "Link against alternative zlib implementation" OFF)

‎test/benchmarks/benchmark_slidehash.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ class slide_hash: public benchmark::Fixture {
5555
void TearDown(const ::benchmark::State& state) {
5656
zng_free(l0);
5757
zng_free(l1);
58+
free(deflate_state);
5859
}
5960
};
6061

0 commit comments

Comments
 (0)