Skip to content

Commit 1a45a14

Browse files
Dead2Dead2
committed
Add AVX512 version of compare256
1 parent e4a31e0 commit 1a45a14

8 files changed

Lines changed: 130 additions & 3 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1041,6 +1041,8 @@ if(WITH_OPTIM)
10411041
add_feature_info(AVX512_ADLER32 1 "Support AVX512-accelerated adler32, using \"${AVX512FLAG}\"")
10421042
list(APPEND AVX512_SRCS ${ARCHDIR}/chunkset_avx512.c)
10431043
add_feature_info(AVX512_CHUNKSET 1 "Support AVX512 optimized chunkset, using \"${AVX512FLAG}\"")
1044+
list(APPEND AVX512_SRCS ${ARCHDIR}/compare256_avx512.c)
1045+
add_feature_info(AVX512_COMPARE256 1 "Support AVX512 optimized compare256, using \"${AVX512FLAG}\"")
10441046
list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/adler32_avx512_p.h)
10451047
list(APPEND ZLIB_ARCH_SRCS ${AVX512_SRCS})
10461048
set_property(SOURCE ${AVX512_SRCS} PROPERTY COMPILE_FLAGS "${AVX512FLAG} ${NOLTOFLAG}")

‎arch/x86/Makefile.in‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ all: \
3636
chunkset_ssse3.o chunkset_ssse3.lo \
3737
chorba_sse2.o chorba_sse2.lo \
3838
compare256_avx2.o compare256_avx2.lo \
39+
compare256_avx512.o compare256_avx512.lo \
3940
compare256_sse2.o compare256_sse2.lo \
4041
crc32_pclmulqdq.o crc32_pclmulqdq.lo \
4142
crc32_vpclmulqdq.o crc32_vpclmulqdq.lo \
@@ -84,6 +85,12 @@ compare256_avx2.o:
8485
compare256_avx2.lo:
8586
$(CC) $(SFLAGS) $(AVX2FLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_avx2.c
8687

88+
compare256_avx512.o:
89+
$(CC) $(CFLAGS) $(AVX512FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_avx512.c
90+
91+
compare256_avx512.lo:
92+
$(CC) $(SFLAGS) $(AVX512FLAG) $(NOLTOFLAG) -DPIC $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_avx512.c
93+
8794
compare256_sse2.o:
8895
$(CC) $(CFLAGS) $(SSE2FLAG) $(NOLTOFLAG) $(INCLUDES) -c -o $@ $(SRCDIR)/compare256_sse2.c
8996

‎arch/x86/compare256_avx512.c‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
/* compare256_avx512.c -- AVX512 version of compare256
2+
* Copyright (C) 2025 Hans Kristian Rosbach
3+
* Based on AVX2 implementation by Mika T. Lindqvist
4+
* For conditions of distribution and use, see copyright notice in zlib.h
5+
*/
6+
7+
#include "zbuild.h"
8+
#include "zmemory.h"
9+
#include "deflate.h"
10+
#include "fallback_builtins.h"
11+
12+
#if defined(X86_AVX512) && defined(HAVE_BUILTIN_CTZLL)
13+
14+
#include <immintrin.h>
15+
#ifdef _MSC_VER
16+
# include <nmmintrin.h>
17+
#endif
18+
19+
static inline uint32_t compare256_avx512_static(const uint8_t *src0, const uint8_t *src1) {
20+
__m256i ymm_src0_1, ymm_src1_1;
21+
__m256i ymm_src0_2, ymm_src1_2;
22+
__m512i zmm_src0_3, zmm_src1_3;
23+
__m512i zmm_src0_4, zmm_src1_4;
24+
__m512i zmm_src0_5, zmm_src1_5;
25+
uint32_t mask_1, mask_2;
26+
uint64_t mask_3, mask_4, mask_5;
27+
28+
// First do a couple 32byte rounds before increasing to 64bytes, this reduces
29+
// the penalty for the short matches, and those are usually the most common ones.
30+
31+
// 32 bytes
32+
ymm_src0_1 = _mm256_loadu_si256((__m256i*)src0);
33+
ymm_src1_1 = _mm256_loadu_si256((__m256i*)src1);
34+
mask_1 = _mm256_cmpeq_epu8_mask(ymm_src0_1, ymm_src1_1);
35+
if (mask_1 != 0xFFFFFFFF) {
36+
uint32_t match_byte = (uint32_t)__builtin_ctz(~mask_1); /* Invert bits so identical = 0 */
37+
return match_byte;
38+
}
39+
40+
// 32 bytes
41+
ymm_src0_2 = _mm256_loadu_si256((__m256i*)(src0 + 32));
42+
ymm_src1_2 = _mm256_loadu_si256((__m256i*)(src1 + 32));
43+
mask_2 = _mm256_cmpeq_epu8_mask(ymm_src0_2, ymm_src1_2);
44+
if (mask_2 != 0xFFFFFFFF) {
45+
uint32_t match_byte = (uint32_t)__builtin_ctz(~mask_2); /* Invert bits so identical = 0 */
46+
return 32 + match_byte;
47+
}
48+
49+
// 64 bytes
50+
zmm_src0_3 = _mm512_loadu_si512((__m512i*)(src0 + 64));
51+
zmm_src1_3 = _mm512_loadu_si512((__m512i*)(src1 + 64));
52+
mask_3 = _mm512_cmpeq_epu8_mask(zmm_src0_3, zmm_src1_3);
53+
if (mask_3 != 0xFFFFFFFFFFFFFFFF) {
54+
uint32_t match_byte = (uint32_t)__builtin_ctzll(~mask_3);
55+
return 64 + match_byte;
56+
}
57+
58+
// 64 bytes
59+
zmm_src0_4 = _mm512_loadu_si512((__m512i*)(src0 + 128));
60+
zmm_src1_4 = _mm512_loadu_si512((__m512i*)(src1 + 128));
61+
mask_4 = _mm512_cmpeq_epu8_mask(zmm_src0_4, zmm_src1_4);
62+
if (mask_4 != 0xFFFFFFFFFFFFFFFF) {
63+
uint32_t match_byte = (uint32_t)__builtin_ctzll(~mask_4);
64+
return 128 + match_byte;
65+
}
66+
67+
// 64 bytes
68+
zmm_src0_5 = _mm512_loadu_si512((__m512i*)(src0 + 192));
69+
zmm_src1_5 = _mm512_loadu_si512((__m512i*)(src1 + 192));
70+
mask_5 = _mm512_cmpeq_epu8_mask(zmm_src0_5, zmm_src1_5);
71+
if (mask_5 != 0xFFFFFFFFFFFFFFFF) {
72+
uint32_t match_byte = (uint32_t)__builtin_ctzll(~mask_5);
73+
return 192 + match_byte;
74+
}
75+
76+
return 256;
77+
}
78+
79+
Z_INTERNAL uint32_t compare256_avx512(const uint8_t *src0, const uint8_t *src1) {
80+
return compare256_avx512_static(src0, src1);
81+
}
82+
83+
#define LONGEST_MATCH longest_match_avx512
84+
#define COMPARE256 compare256_avx512_static
85+
86+
#include "match_tpl.h"
87+
88+
#define LONGEST_MATCH_SLOW
89+
#define LONGEST_MATCH longest_match_slow_avx512
90+
#define COMPARE256 compare256_avx512_static
91+
92+
#include "match_tpl.h"
93+
94+
#endif

‎arch/x86/x86_functions.h‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ uint32_t adler32_fold_copy_avx512(uint32_t adler, uint8_t *dst, const uint8_t *s
6060
uint32_t chunksize_avx512(void);
6161
uint8_t* chunkmemset_safe_avx512(uint8_t *out, uint8_t *from, unsigned len, unsigned left);
6262
void inflate_fast_avx512(PREFIX3(stream)* strm, uint32_t start);
63+
# ifdef HAVE_BUILTIN_CTZLL
64+
uint32_t compare256_avx512(const uint8_t *src0, const uint8_t *src1);
65+
uint32_t longest_match_avx512(deflate_state *const s, Pos cur_match);
66+
uint32_t longest_match_slow_avx512(deflate_state *const s, Pos cur_match);
67+
# endif
6368
#endif
6469
#ifdef X86_AVX512VNNI
6570
uint32_t adler32_avx512_vnni(uint32_t adler, const uint8_t *buf, size_t len);
@@ -169,6 +174,14 @@ uint32_t crc32_vpclmulqdq(uint32_t crc32, const uint8_t *buf, size_t len);
169174
# define native_chunksize chunksize_avx512
170175
# undef native_inflate_fast
171176
# define native_inflate_fast inflate_fast_avx512
177+
# ifdef HAVE_BUILTIN_CTZLL
178+
# undef native_compare256
179+
# define native_compare256 compare256_avx512
180+
# undef native_longest_match
181+
# define native_longest_match longest_match_avx512
182+
# undef native_longest_match_slow
183+
# define native_longest_match_slow longest_match_slow_avx512
184+
# endif
172185
// X86 - AVX512 (VNNI)
173186
# if defined(X86_AVX512VNNI) && defined(__AVX512VNNI__)
174187
# undef native_adler32

‎configure‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1694,8 +1694,8 @@ case "${ARCH}" in
16941694
if test ${HAVE_AVX512_INTRIN} -eq 1; then
16951695
CFLAGS="${CFLAGS} -DX86_AVX512"
16961696
SFLAGS="${SFLAGS} -DX86_AVX512"
1697-
ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_avx512.o chunkset_avx512.o"
1698-
ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_avx512.lo chunkset_avx512.lo"
1697+
ARCH_STATIC_OBJS="${ARCH_STATIC_OBJS} adler32_avx512.o chunkset_avx512.o compare256_avx512.o"
1698+
ARCH_SHARED_OBJS="${ARCH_SHARED_OBJS} adler32_avx512.lo chunkset_avx512.lo compare256_avx512.lo"
16991699
fi
17001700

17011701
check_mtune_cascadelake_compiler_flag

‎functable.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,11 @@ static void init_functable(void) {
139139
ft.chunkmemset_safe = &chunkmemset_safe_avx512;
140140
ft.chunksize = &chunksize_avx512;
141141
ft.inflate_fast = &inflate_fast_avx512;
142+
# ifdef HAVE_BUILTIN_CTZLL
143+
ft.compare256 = &compare256_avx512;
144+
ft.longest_match = &longest_match_avx512;
145+
ft.longest_match_slow = &longest_match_slow_avx512;
146+
# endif
142147
}
143148
#endif
144149
#ifdef X86_AVX512VNNI

‎test/benchmarks/benchmark_compare256.cc‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ class compare256: public benchmark::Fixture {
5959
} \
6060
Bench(state, fptr); \
6161
} \
62-
BENCHMARK_REGISTER_F(compare256, name)->Range(1, MAX_COMPARE_SIZE);
62+
BENCHMARK_REGISTER_F(compare256, name)->Arg(1)->Arg(8)->Arg(32)->Arg(96)->Arg(160)->Arg(256);
6363

6464
#ifdef DISABLE_RUNTIME_CPU_DETECTION
6565
BENCHMARK_COMPARE256(native, native_compare256, 1);
@@ -80,6 +80,9 @@ BENCHMARK_COMPARE256(sse2, compare256_sse2, test_cpu_features.x86.has_sse2);
8080
#if defined(X86_AVX2) && defined(HAVE_BUILTIN_CTZ)
8181
BENCHMARK_COMPARE256(avx2, compare256_avx2, test_cpu_features.x86.has_avx2);
8282
#endif
83+
#if defined(X86_AVX512) && defined(HAVE_BUILTIN_CTZLL)
84+
BENCHMARK_COMPARE256(avx512, compare256_avx512, test_cpu_features.x86.has_avx512_common);
85+
#endif
8386
#if defined(ARM_NEON) && defined(HAVE_BUILTIN_CTZLL)
8487
BENCHMARK_COMPARE256(neon, compare256_neon, test_cpu_features.arm.has_neon);
8588
#endif

‎test/test_compare256.cc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,9 @@ TEST_COMPARE256(sse2, compare256_sse2, test_cpu_features.x86.has_sse2)
7979
#if defined(X86_AVX2) && defined(HAVE_BUILTIN_CTZ)
8080
TEST_COMPARE256(avx2, compare256_avx2, test_cpu_features.x86.has_avx2)
8181
#endif
82+
#if defined(X86_AVX512) && defined(HAVE_BUILTIN_CTZLL)
83+
TEST_COMPARE256(avx512, compare256_avx512, test_cpu_features.x86.has_avx512_common)
84+
#endif
8285
#if defined(ARM_NEON) && defined(HAVE_BUILTIN_CTZLL)
8386
TEST_COMPARE256(neon, compare256_neon, test_cpu_features.arm.has_neon)
8487
#endif

0 commit comments

Comments
 (0)