| diff --git a/crc32.c b/crc32.c |
| index 9580440c0e6b..9162429cc7b4 100644 |
| --- a/crc32.c |
| +++ b/crc32.c |
| @@ -28,6 +28,8 @@ |
| # endif /* !DYNAMIC_CRC_TABLE */ |
| #endif /* MAKECRCH */ |
| |
| +#include "deflate.h" |
| +#include "x86.h" |
| #include "zutil.h" /* for STDC and FAR definitions */ |
| |
| /* Definitions for doing the crc four data bytes at a time. */ |
| @@ -440,3 +442,28 @@ uLong ZEXPORT crc32_combine64(crc1, crc2, len2) |
| { |
| return crc32_combine_(crc1, crc2, len2); |
| } |
| + |
| +ZLIB_INTERNAL void crc_reset(deflate_state *const s) |
| +{ |
| + if (x86_cpu_enable_simd) { |
| + crc_fold_init(s); |
| + return; |
| + } |
| + s->strm->adler = crc32(0L, Z_NULL, 0); |
| +} |
| + |
| +ZLIB_INTERNAL void crc_finalize(deflate_state *const s) |
| +{ |
| + if (x86_cpu_enable_simd) |
| + s->strm->adler = crc_fold_512to32(s); |
| +} |
| + |
| +ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) |
| +{ |
| + if (x86_cpu_enable_simd) { |
| + crc_fold_copy(strm->state, dst, strm->next_in, size); |
| + return; |
| + } |
| + zmemcpy(dst, strm->next_in, size); |
| + strm->adler = crc32(strm->adler, dst, size); |
| +} |
| diff --git a/crc_folding.c b/crc_folding.c |
| new file mode 100644 |
| index 000000000000..48d77744aaf4 |
| --- /dev/null |
| +++ b/crc_folding.c |
| @@ -0,0 +1,493 @@ |
| +/* |
| + * Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ |
| + * instruction. |
| + * |
| + * A white paper describing this algorithm can be found at: |
| + * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf |
| + * |
| + * Copyright (C) 2013 Intel Corporation. All rights reserved. |
| + * Authors: |
| + * Wajdi Feghali <wajdi.k.feghali@intel.com> |
| + * Jim Guilford <james.guilford@intel.com> |
| + * Vinodh Gopal <vinodh.gopal@intel.com> |
| + * Erdinc Ozturk <erdinc.ozturk@intel.com> |
| + * Jim Kukunas <james.t.kukunas@linux.intel.com> |
| + * |
| + * For conditions of distribution and use, see copyright notice in zlib.h |
| + */ |
| + |
| +#include "deflate.h" |
| + |
| +#include <inttypes.h> |
| +#include <emmintrin.h> |
| +#include <immintrin.h> |
| +#include <wmmintrin.h> |
| + |
| +#define CRC_LOAD(s) \ |
| + do { \ |
| + __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ |
| + __m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1);\ |
| + __m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2);\ |
| + __m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3);\ |
| + __m128i xmm_crc_part = _mm_loadu_si128((__m128i *)s->crc0 + 4); |
| + |
| +#define CRC_SAVE(s) \ |
| + _mm_storeu_si128((__m128i *)s->crc0 + 0, xmm_crc0);\ |
| + _mm_storeu_si128((__m128i *)s->crc0 + 1, xmm_crc1);\ |
| + _mm_storeu_si128((__m128i *)s->crc0 + 2, xmm_crc2);\ |
| + _mm_storeu_si128((__m128i *)s->crc0 + 3, xmm_crc3);\ |
| + _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ |
| + } while (0); |
| + |
| +ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) |
| +{ |
| + CRC_LOAD(s) |
| + |
| + xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); |
| + xmm_crc1 = _mm_setzero_si128(); |
| + xmm_crc2 = _mm_setzero_si128(); |
| + xmm_crc3 = _mm_setzero_si128(); |
| + |
| + CRC_SAVE(s) |
| + |
| + s->strm->adler = 0; |
| +} |
| + |
| +local void fold_1(deflate_state *const s, |
| + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| + __m128i *xmm_crc2, __m128i *xmm_crc3) |
| +{ |
| + const __m128i xmm_fold4 = _mm_set_epi32( |
| + 0x00000001, 0x54442bd4, |
| + 0x00000001, 0xc6e41596); |
| + |
| + __m128i x_tmp3; |
| + __m128 ps_crc0, ps_crc3, ps_res; |
| + |
| + x_tmp3 = *xmm_crc3; |
| + |
| + *xmm_crc3 = *xmm_crc0; |
| + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); |
| + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); |
| + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); |
| + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| + ps_res = _mm_xor_ps(ps_crc0, ps_crc3); |
| + |
| + *xmm_crc0 = *xmm_crc1; |
| + *xmm_crc1 = *xmm_crc2; |
| + *xmm_crc2 = x_tmp3; |
| + *xmm_crc3 = _mm_castps_si128(ps_res); |
| +} |
| + |
| +local void fold_2(deflate_state *const s, |
| + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| + __m128i *xmm_crc2, __m128i *xmm_crc3) |
| +{ |
| + const __m128i xmm_fold4 = _mm_set_epi32( |
| + 0x00000001, 0x54442bd4, |
| + 0x00000001, 0xc6e41596); |
| + |
| + __m128i x_tmp3, x_tmp2; |
| + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20; |
| + |
| + x_tmp3 = *xmm_crc3; |
| + x_tmp2 = *xmm_crc2; |
| + |
| + *xmm_crc3 = *xmm_crc1; |
| + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); |
| + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); |
| + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); |
| + ps_res31= _mm_xor_ps(ps_crc3, ps_crc1); |
| + |
| + *xmm_crc2 = *xmm_crc0; |
| + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); |
| + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); |
| + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); |
| + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); |
| + ps_res20= _mm_xor_ps(ps_crc0, ps_crc2); |
| + |
| + *xmm_crc0 = x_tmp2; |
| + *xmm_crc1 = x_tmp3; |
| + *xmm_crc2 = _mm_castps_si128(ps_res20); |
| + *xmm_crc3 = _mm_castps_si128(ps_res31); |
| +} |
| + |
| +local void fold_3(deflate_state *const s, |
| + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| + __m128i *xmm_crc2, __m128i *xmm_crc3) |
| +{ |
| + const __m128i xmm_fold4 = _mm_set_epi32( |
| + 0x00000001, 0x54442bd4, |
| + 0x00000001, 0xc6e41596); |
| + |
| + __m128i x_tmp3; |
| + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10; |
| + |
| + x_tmp3 = *xmm_crc3; |
| + |
| + *xmm_crc3 = *xmm_crc2; |
| + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); |
| + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); |
| + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); |
| + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| + ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3); |
| + |
| + *xmm_crc2 = *xmm_crc1; |
| + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); |
| + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); |
| + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); |
| + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); |
| + ps_res21= _mm_xor_ps(ps_crc1, ps_crc2); |
| + |
| + *xmm_crc1 = *xmm_crc0; |
| + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); |
| + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); |
| + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); |
| + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); |
| + ps_res10= _mm_xor_ps(ps_crc0, ps_crc1); |
| + |
| + *xmm_crc0 = x_tmp3; |
| + *xmm_crc1 = _mm_castps_si128(ps_res10); |
| + *xmm_crc2 = _mm_castps_si128(ps_res21); |
| + *xmm_crc3 = _mm_castps_si128(ps_res32); |
| +} |
| + |
| +local void fold_4(deflate_state *const s, |
| + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| + __m128i *xmm_crc2, __m128i *xmm_crc3) |
| +{ |
| + const __m128i xmm_fold4 = _mm_set_epi32( |
| + 0x00000001, 0x54442bd4, |
| + 0x00000001, 0xc6e41596); |
| + |
| + __m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3; |
| + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3; |
| + __m128 ps_t0, ps_t1, ps_t2, ps_t3; |
| + __m128 ps_res0, ps_res1, ps_res2, ps_res3; |
| + |
| + x_tmp0 = *xmm_crc0; |
| + x_tmp1 = *xmm_crc1; |
| + x_tmp2 = *xmm_crc2; |
| + x_tmp3 = *xmm_crc3; |
| + |
| + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); |
| + x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10); |
| + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); |
| + ps_t0 = _mm_castsi128_ps(x_tmp0); |
| + ps_res0 = _mm_xor_ps(ps_crc0, ps_t0); |
| + |
| + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); |
| + x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10); |
| + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); |
| + ps_t1 = _mm_castsi128_ps(x_tmp1); |
| + ps_res1 = _mm_xor_ps(ps_crc1, ps_t1); |
| + |
| + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); |
| + x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10); |
| + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); |
| + ps_t2 = _mm_castsi128_ps(x_tmp2); |
| + ps_res2 = _mm_xor_ps(ps_crc2, ps_t2); |
| + |
| + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); |
| + x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10); |
| + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| + ps_t3 = _mm_castsi128_ps(x_tmp3); |
| + ps_res3 = _mm_xor_ps(ps_crc3, ps_t3); |
| + |
| + *xmm_crc0 = _mm_castps_si128(ps_res0); |
| + *xmm_crc1 = _mm_castps_si128(ps_res1); |
| + *xmm_crc2 = _mm_castps_si128(ps_res2); |
| + *xmm_crc3 = _mm_castps_si128(ps_res3); |
| +} |
| + |
| +local const unsigned zalign(32) pshufb_shf_table[60] = { |
| + 0x84838281,0x88878685,0x8c8b8a89,0x008f8e8d, /* shl 15 (16 - 1)/shr1 */ |
| + 0x85848382,0x89888786,0x8d8c8b8a,0x01008f8e, /* shl 14 (16 - 3)/shr2 */ |
| + 0x86858483,0x8a898887,0x8e8d8c8b,0x0201008f, /* shl 13 (16 - 4)/shr3 */ |
| + 0x87868584,0x8b8a8988,0x8f8e8d8c,0x03020100, /* shl 12 (16 - 4)/shr4 */ |
| + 0x88878685,0x8c8b8a89,0x008f8e8d,0x04030201, /* shl 11 (16 - 5)/shr5 */ |
| + 0x89888786,0x8d8c8b8a,0x01008f8e,0x05040302, /* shl 10 (16 - 6)/shr6 */ |
| + 0x8a898887,0x8e8d8c8b,0x0201008f,0x06050403, /* shl 9 (16 - 7)/shr7 */ |
| + 0x8b8a8988,0x8f8e8d8c,0x03020100,0x07060504, /* shl 8 (16 - 8)/shr8 */ |
| + 0x8c8b8a89,0x008f8e8d,0x04030201,0x08070605, /* shl 7 (16 - 9)/shr9 */ |
| + 0x8d8c8b8a,0x01008f8e,0x05040302,0x09080706, /* shl 6 (16 -10)/shr10*/ |
| + 0x8e8d8c8b,0x0201008f,0x06050403,0x0a090807, /* shl 5 (16 -11)/shr11*/ |
| + 0x8f8e8d8c,0x03020100,0x07060504,0x0b0a0908, /* shl 4 (16 -12)/shr12*/ |
| + 0x008f8e8d,0x04030201,0x08070605,0x0c0b0a09, /* shl 3 (16 -13)/shr13*/ |
| + 0x01008f8e,0x05040302,0x09080706,0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/ |
| + 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ |
| +}; |
| + |
| +local void partial_fold(deflate_state *const s, const size_t len, |
| + __m128i *xmm_crc0, __m128i *xmm_crc1, |
| + __m128i *xmm_crc2, __m128i *xmm_crc3, |
| + __m128i *xmm_crc_part) |
| +{ |
| + |
| + const __m128i xmm_fold4 = _mm_set_epi32( |
| + 0x00000001, 0x54442bd4, |
| + 0x00000001, 0xc6e41596); |
| + const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080); |
| + |
| + __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3; |
| + __m128i xmm_a0_0, xmm_a0_1; |
| + __m128 ps_crc3, psa0_0, psa0_1, ps_res; |
| + |
| + xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1)); |
| + xmm_shr = xmm_shl; |
| + xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3); |
| + |
| + xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl); |
| + |
| + *xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr); |
| + xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl); |
| + *xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1); |
| + |
| + *xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr); |
| + xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl); |
| + *xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2); |
| + |
| + *xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr); |
| + xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl); |
| + *xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3); |
| + |
| + *xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr); |
| + *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl); |
| + *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part); |
| + |
| + xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10); |
| + xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01); |
| + |
| + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); |
| + psa0_0 = _mm_castsi128_ps(xmm_a0_0); |
| + psa0_1 = _mm_castsi128_ps(xmm_a0_1); |
| + |
| + ps_res = _mm_xor_ps(ps_crc3, psa0_0); |
| + ps_res = _mm_xor_ps(ps_res, psa0_1); |
| + |
| + *xmm_crc3 = _mm_castps_si128(ps_res); |
| +} |
| + |
| +ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, |
| + unsigned char *dst, const unsigned char *src, long len) |
| +{ |
| + unsigned long algn_diff; |
| + __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; |
| + |
| + CRC_LOAD(s) |
| + |
| + if (len < 16) { |
| + if (len == 0) |
| + return; |
| + goto partial; |
| + } |
| + |
| + algn_diff = 0 - (uintptr_t)src & 0xF; |
| + if (algn_diff) { |
| + xmm_crc_part = _mm_loadu_si128((__m128i *)src); |
| + _mm_storeu_si128((__m128i *)dst, xmm_crc_part); |
| + |
| + dst += algn_diff; |
| + src += algn_diff; |
| + len -= algn_diff; |
| + |
| + partial_fold(s, algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, |
| + &xmm_crc_part); |
| + } |
| + |
| + while ((len -= 64) >= 0) { |
| + xmm_t0 = _mm_load_si128((__m128i *)src); |
| + xmm_t1 = _mm_load_si128((__m128i *)src + 1); |
| + xmm_t2 = _mm_load_si128((__m128i *)src + 2); |
| + xmm_t3 = _mm_load_si128((__m128i *)src + 3); |
| + |
| + fold_4(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); |
| + |
| + _mm_storeu_si128((__m128i *)dst, xmm_t0); |
| + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); |
| + _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); |
| + _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); |
| + |
| + xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); |
| + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); |
| + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); |
| + |
| + src += 64; |
| + dst += 64; |
| + } |
| + |
| + /* |
| + * len = num bytes left - 64 |
| + */ |
| + if (len + 16 >= 0) { |
| + len += 16; |
| + |
| + xmm_t0 = _mm_load_si128((__m128i *)src); |
| + xmm_t1 = _mm_load_si128((__m128i *)src + 1); |
| + xmm_t2 = _mm_load_si128((__m128i *)src + 2); |
| + |
| + fold_3(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); |
| + |
| + _mm_storeu_si128((__m128i *)dst, xmm_t0); |
| + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); |
| + _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); |
| + |
| + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); |
| + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); |
| + |
| + if (len == 0) |
| + goto done; |
| + |
| + dst += 48; |
| + src += 48; |
| + } else if (len + 32 >= 0) { |
| + len += 32; |
| + |
| + xmm_t0 = _mm_load_si128((__m128i *)src); |
| + xmm_t1 = _mm_load_si128((__m128i *)src + 1); |
| + |
| + fold_2(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); |
| + |
| + _mm_storeu_si128((__m128i *)dst, xmm_t0); |
| + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); |
| + |
| + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); |
| + |
| + if (len == 0) |
| + goto done; |
| + |
| + dst += 32; |
| + src += 32; |
| + } else if (len + 48 >= 0) { |
| + len += 48; |
| + |
| + xmm_t0 = _mm_load_si128((__m128i *)src); |
| + |
| + fold_1(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); |
| + |
| + _mm_storeu_si128((__m128i *)dst, xmm_t0); |
| + |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); |
| + |
| + if (len == 0) |
| + goto done; |
| + |
| + dst += 16; |
| + src += 16; |
| + } else { |
| + len += 64; |
| + if (len == 0) |
| + goto done; |
| + } |
| + |
| +partial: |
| + |
| +#if defined(_MSC_VER) |
| + /* VS does not permit the use of _mm_set_epi64x in 32-bit builds */ |
| + { |
| + int32_t parts[4] = {0, 0, 0, 0}; |
| + memcpy(&parts, src, len); |
| + xmm_crc_part = _mm_set_epi32(parts[3], parts[2], parts[1], parts[0]); |
| + } |
| +#else |
| + { |
| + int64_t parts[2] = {0, 0}; |
| + memcpy(&parts, src, len); |
| + xmm_crc_part = _mm_set_epi64x(parts[1], parts[0]); |
| + } |
| +#endif |
| + |
| + _mm_storeu_si128((__m128i *)dst, xmm_crc_part); |
| + partial_fold(s, len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, |
| + &xmm_crc_part); |
| +done: |
| + CRC_SAVE(s) |
| +} |
| + |
| +local const unsigned zalign(16) crc_k[] = { |
| + 0xccaa009e, 0x00000000, /* rk1 */ |
| + 0x751997d0, 0x00000001, /* rk2 */ |
| + 0xccaa009e, 0x00000000, /* rk5 */ |
| + 0x63cd6124, 0x00000001, /* rk6 */ |
| + 0xf7011640, 0x00000001, /* rk7 */ |
| + 0xdb710640, 0x00000001 /* rk8 */ |
| +}; |
| + |
| +local const unsigned zalign(16) crc_mask[4] = { |
| + 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 |
| +}; |
| + |
| +local const unsigned zalign(16) crc_mask2[4] = { |
| + 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF |
| +}; |
| + |
| +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) |
| +{ |
| + const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); |
| + const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2); |
| + |
| + unsigned crc; |
| + __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; |
| + |
| + CRC_LOAD(s) |
| + |
| + /* |
| + * k1 |
| + */ |
| + crc_fold = _mm_load_si128((__m128i *)crc_k); |
| + |
| + x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); |
| + xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); |
| + xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0); |
| + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0); |
| + |
| + x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); |
| + xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); |
| + xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1); |
| + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1); |
| + |
| + x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); |
| + xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); |
| + |
| + /* |
| + * k5 |
| + */ |
| + crc_fold = _mm_load_si128((__m128i *)crc_k + 1); |
| + |
| + xmm_crc0 = xmm_crc3; |
| + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); |
| + xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); |
| + |
| + xmm_crc0 = xmm_crc3; |
| + xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); |
| + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); |
| + xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2); |
| + |
| + /* |
| + * k7 |
| + */ |
| + xmm_crc1 = xmm_crc3; |
| + xmm_crc2 = xmm_crc3; |
| + crc_fold = _mm_load_si128((__m128i *)crc_k + 2); |
| + |
| + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); |
| + xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask); |
| + |
| + xmm_crc2 = xmm_crc3; |
| + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); |
| + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); |
| + |
| + crc = _mm_extract_epi32(xmm_crc3, 2); |
| + return ~crc; |
| + CRC_SAVE(s) |
| +} |
| diff --git a/deflate.c b/deflate.c |
| index 1ec761448de9..aa0c9c67a6dc 100644 |
| --- a/deflate.c |
| +++ b/deflate.c |
| @@ -48,8 +48,9 @@ |
| */ |
| |
| /* @(#) $Id$ */ |
| - |
| +#include <assert.h> |
| #include "deflate.h" |
| +#include "x86.h" |
| |
| const char deflate_copyright[] = |
| " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; |
| @@ -86,7 +87,7 @@ local block_state deflate_huff OF((deflate_state *s, int flush)); |
| local void lm_init OF((deflate_state *s)); |
| local void putShortMSB OF((deflate_state *s, uInt b)); |
| local void flush_pending OF((z_streamp strm)); |
| -local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); |
| +unsigned ZLIB_INTERNAL deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); |
| #ifdef ASMV |
| # pragma message("Assembler code may have bugs -- use at your own risk") |
| void match_init OF((void)); /* asm code initialization */ |
| @@ -100,6 +101,20 @@ local void check_match OF((deflate_state *s, IPos start, IPos match, |
| int length)); |
| #endif |
| |
| +/* From crc32.c */ |
| +extern void ZLIB_INTERNAL crc_reset(deflate_state *const s); |
| +extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s); |
| +extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size); |
| + |
| +#ifdef _MSC_VER |
| +#define INLINE __inline |
| +#else |
| +#define INLINE inline |
| +#endif |
| + |
| +/* Inline optimisation */ |
| +local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str); |
| + |
| /* =========================================================================== |
| * Local data |
| */ |
| @@ -162,7 +177,6 @@ local const config configuration_table[10] = { |
| */ |
| #define UPDATE_HASH(s,h,c) (h = (((h)<<s->hash_shift) ^ (c)) & s->hash_mask) |
| |
| - |
| /* =========================================================================== |
| * Insert string str in the dictionary and set match_head to the previous head |
| * of the hash chain (the most recent string with same hash key). Return |
| @@ -173,17 +187,28 @@ local const config configuration_table[10] = { |
| * characters and the first MIN_MATCH bytes of str are valid (except for |
| * the last MIN_MATCH-1 bytes of the input file). |
| */ |
| +local INLINE Pos insert_string_c(deflate_state *const s, const Pos str) |
| +{ |
| + Pos ret; |
| + |
| + UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]); |
| #ifdef FASTEST |
| -#define INSERT_STRING(s, str, match_head) \ |
| - (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ |
| - match_head = s->head[s->ins_h], \ |
| - s->head[s->ins_h] = (Pos)(str)) |
| + ret = s->head[s->ins_h]; |
| #else |
| -#define INSERT_STRING(s, str, match_head) \ |
| - (UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH-1)]), \ |
| - match_head = s->prev[(str) & s->w_mask] = s->head[s->ins_h], \ |
| - s->head[s->ins_h] = (Pos)(str)) |
| + ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; |
| #endif |
| + s->head[s->ins_h] = str; |
| + |
| + return ret; |
| +} |
| + |
| +local INLINE Pos insert_string(deflate_state *const s, const Pos str) |
| +{ |
| + if (x86_cpu_enable_simd) |
| + return insert_string_sse(s, str); |
| + return insert_string_c(s, str); |
| +} |
| + |
| |
| /* =========================================================================== |
| * Initialize the hash table (avoiding 64K overflow for 16 bit systems). |
| @@ -248,6 +273,7 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, |
| const char *version; |
| int stream_size; |
| { |
| + unsigned window_padding = 8; |
| deflate_state *s; |
| int wrap = 1; |
| static const char my_version[] = ZLIB_VERSION; |
| @@ -257,6 +283,8 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, |
| * output size for (length,distance) codes is <= 24 bits. |
| */ |
| |
| + x86_check_features(); |
| + |
| if (version == Z_NULL || version[0] != my_version[0] || |
| stream_size != sizeof(z_stream)) { |
| return Z_VERSION_ERROR; |
| @@ -313,12 +341,19 @@ int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, |
| s->w_size = 1 << s->w_bits; |
| s->w_mask = s->w_size - 1; |
| |
| - s->hash_bits = (uInt)memLevel + 7; |
| + if (x86_cpu_enable_simd) { |
| + s->hash_bits = 15; |
| + } else { |
| + s->hash_bits = memLevel + 7; |
| + } |
| + |
| s->hash_size = 1 << s->hash_bits; |
| s->hash_mask = s->hash_size - 1; |
| s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); |
| |
| - s->window = (Bytef *) ZALLOC(strm, s->w_size, 2*sizeof(Byte)); |
| + s->window = (Bytef *) ZALLOC(strm, |
| + s->w_size + window_padding, |
| + 2*sizeof(Byte)); |
| s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); |
| s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); |
| |
| @@ -418,11 +453,7 @@ int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) |
| str = s->strstart; |
| n = s->lookahead - (MIN_MATCH-1); |
| do { |
| - UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); |
| -#ifndef FASTEST |
| - s->prev[str & s->w_mask] = s->head[s->ins_h]; |
| -#endif |
| - s->head[s->ins_h] = (Pos)str; |
| + insert_string(s, str); |
| str++; |
| } while (--n); |
| s->strstart = str; |
| @@ -848,7 +879,7 @@ int ZEXPORT deflate (strm, flush) |
| #ifdef GZIP |
| if (s->status == GZIP_STATE) { |
| /* gzip header */ |
| - strm->adler = crc32(0L, Z_NULL, 0); |
| + crc_reset(s); |
| put_byte(s, 31); |
| put_byte(s, 139); |
| put_byte(s, 8); |
| @@ -1049,6 +1080,7 @@ int ZEXPORT deflate (strm, flush) |
| /* Write the trailer */ |
| #ifdef GZIP |
| if (s->wrap == 2) { |
| + crc_finalize(s); |
| put_byte(s, (Byte)(strm->adler & 0xff)); |
| put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); |
| put_byte(s, (Byte)((strm->adler >> 16) & 0xff)); |
| @@ -1161,7 +1193,7 @@ int ZEXPORT deflateCopy (dest, source) |
| * allocating a large strm->next_in buffer and copying from it. |
| * (See also flush_pending()). |
| */ |
| -local unsigned read_buf(strm, buf, size) |
| +ZLIB_INTERNAL unsigned deflate_read_buf(strm, buf, size) |
| z_streamp strm; |
| Bytef *buf; |
| unsigned size; |
| @@ -1173,15 +1205,16 @@ local unsigned read_buf(strm, buf, size) |
| |
| strm->avail_in -= len; |
| |
| - zmemcpy(buf, strm->next_in, len); |
| - if (strm->state->wrap == 1) { |
| - strm->adler = adler32(strm->adler, buf, len); |
| - } |
| #ifdef GZIP |
| - else if (strm->state->wrap == 2) { |
| - strm->adler = crc32(strm->adler, buf, len); |
| - } |
| + if (strm->state->wrap == 2) |
| + copy_with_crc(strm, buf, len); |
| + else |
| #endif |
| + { |
| + zmemcpy(buf, strm->next_in, len); |
| + if (strm->state->wrap == 1) |
| + strm->adler = adler32(strm->adler, buf, len); |
| + } |
| strm->next_in += len; |
| strm->total_in += len; |
| |
| @@ -1479,7 +1512,19 @@ local void check_match(s, start, match, length) |
| * performed for at least two bytes (required for the zip translate_eol |
| * option -- not supported here). |
| */ |
| -local void fill_window(s) |
| +local void fill_window_c(deflate_state *s); |
| + |
| +local void fill_window(deflate_state *s) |
| +{ |
| + if (x86_cpu_enable_simd) { |
| + fill_window_sse(s); |
| + return; |
| + } |
| + |
| + fill_window_c(s); |
| +} |
| + |
| +local void fill_window_c(s) |
| deflate_state *s; |
| { |
| unsigned n; |
| @@ -1847,7 +1892,7 @@ local block_state deflate_fast(s, flush) |
| */ |
| hash_head = NIL; |
| if (s->lookahead >= MIN_MATCH) { |
| - INSERT_STRING(s, s->strstart, hash_head); |
| + hash_head = insert_string(s, s->strstart); |
| } |
| |
| /* Find the longest match, discarding those <= prev_length. |
| @@ -1878,7 +1923,7 @@ local block_state deflate_fast(s, flush) |
| s->match_length--; /* string at strstart already in table */ |
| do { |
| s->strstart++; |
| - INSERT_STRING(s, s->strstart, hash_head); |
| + hash_head = insert_string(s, s->strstart); |
| /* strstart never exceeds WSIZE-MAX_MATCH, so there are |
| * always MIN_MATCH bytes ahead. |
| */ |
| @@ -1950,7 +1995,7 @@ local block_state deflate_slow(s, flush) |
| */ |
| hash_head = NIL; |
| if (s->lookahead >= MIN_MATCH) { |
| - INSERT_STRING(s, s->strstart, hash_head); |
| + hash_head = insert_string(s, s->strstart); |
| } |
| |
| /* Find the longest match, discarding those <= prev_length. |
| @@ -2001,7 +2046,7 @@ local block_state deflate_slow(s, flush) |
| s->prev_length -= 2; |
| do { |
| if (++s->strstart <= max_insert) { |
| - INSERT_STRING(s, s->strstart, hash_head); |
| + hash_head = insert_string(s, s->strstart); |
| } |
| } while (--s->prev_length != 0); |
| s->match_available = 0; |
| @@ -2161,3 +2206,37 @@ local block_state deflate_huff(s, flush) |
| FLUSH_BLOCK(s, 0); |
| return block_done; |
| } |
| + |
| +/* Safe to inline this as GCC/clang will use inline asm and Visual Studio will |
| + * use intrinsic without extra params |
| + */ |
| +local INLINE Pos insert_string_sse(deflate_state *const s, const Pos str) |
| +{ |
| + Pos ret; |
| + unsigned *ip, val, h = 0; |
| + |
| + ip = (unsigned *)&s->window[str]; |
| + val = *ip; |
| + |
| + if (s->level >= 6) |
| + val &= 0xFFFFFF; |
| + |
| +/* Windows clang should use inline asm */ |
| +#if defined(_MSC_VER) && !defined(__clang__) |
| + h = _mm_crc32_u32(h, val); |
| +#elif defined(__i386__) || defined(__amd64__) |
| + __asm__ __volatile__ ( |
| + "crc32 %1,%0\n\t" |
| + : "+r" (h) |
| + : "r" (val) |
| + ); |
| +#else |
| + /* This should never happen */ |
| + assert(0); |
| +#endif |
| + |
| + ret = s->head[h & s->hash_mask]; |
| + s->head[h & s->hash_mask] = str; |
| + s->prev[str & s->w_mask] = ret; |
| + return ret; |
| +} |
| diff --git a/deflate.h b/deflate.h |
| index 23ecdd312bc0..ab56df7663b6 100644 |
| --- a/deflate.h |
| +++ b/deflate.h |
| @@ -109,7 +109,7 @@ typedef struct internal_state { |
| ulg gzindex; /* where in extra, name, or comment */ |
| Byte method; /* can only be DEFLATED */ |
| int last_flush; /* value of flush param for previous deflate call */ |
| - |
| + unsigned zalign(16) crc0[4 * 5]; |
| /* used by deflate.c: */ |
| |
| uInt w_size; /* LZ77 window size (32K by default) */ |
| @@ -346,4 +346,14 @@ void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, |
| flush = _tr_tally(s, distance, length) |
| #endif |
| |
| +/* Functions that are SIMD optimised on x86 */ |
| +void ZLIB_INTERNAL crc_fold_init(deflate_state* const s); |
| +void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s, |
| + unsigned char* dst, |
| + const unsigned char* src, |
| + long len); |
| +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s); |
| + |
| +void ZLIB_INTERNAL fill_window_sse(deflate_state* s); |
| + |
| #endif /* DEFLATE_H */ |
| diff --git a/fill_window_sse.c b/fill_window_sse.c |
| new file mode 100644 |
| index 000000000000..949ccce1ba9c |
| --- /dev/null |
| +++ b/fill_window_sse.c |
| @@ -0,0 +1,177 @@ |
| +/* |
| + * Fill Window with SSE2-optimized hash shifting |
| + * |
| + * Copyright (C) 2013 Intel Corporation |
| + * Authors: |
| + * Arjan van de Ven <arjan@linux.intel.com> |
| + * Jim Kukunas <james.t.kukunas@linux.intel.com> |
| + * |
| + * For conditions of distribution and use, see copyright notice in zlib.h |
| + */ |
| + |
| +#include <immintrin.h> |
| +#include "deflate.h" |
| + |
| +#define UPDATE_HASH(s,h,i) \ |
| + {\ |
| + if (s->level < 6) { \ |
| + h = (3483 * (s->window[i]) +\ |
| + 23081* (s->window[i+1]) +\ |
| + 6954 * (s->window[i+2]) +\ |
| + 20947* (s->window[i+3])) & s->hash_mask;\ |
| + } else {\ |
| + h = (25881* (s->window[i]) +\ |
| + 24674* (s->window[i+1]) +\ |
| + 25811* (s->window[i+2])) & s->hash_mask;\ |
| + }\ |
| + }\ |
| + |
| +extern int deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); |
| + |
| +void fill_window_sse(deflate_state *s) |
| +{ |
| + const __m128i xmm_wsize = _mm_set1_epi16(s->w_size); |
| + |
| + register unsigned n; |
| + register Posf *p; |
| + unsigned more; /* Amount of free space at the end of the window. */ |
| + uInt wsize = s->w_size; |
| + |
| + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); |
| + |
| + do { |
| + more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); |
| + |
| + /* Deal with !@#$% 64K limit: */ |
| + if (sizeof(int) <= 2) { |
| + if (more == 0 && s->strstart == 0 && s->lookahead == 0) { |
| + more = wsize; |
| + |
| + } else if (more == (unsigned)(-1)) { |
| + /* Very unlikely, but possible on 16 bit machine if |
| + * strstart == 0 && lookahead == 1 (input done a byte at time) |
| + */ |
| + more--; |
| + } |
| + } |
| + |
| + /* If the window is almost full and there is insufficient lookahead, |
| + * move the upper half to the lower one to make room in the upper half. |
| + */ |
| + if (s->strstart >= wsize+MAX_DIST(s)) { |
| + |
| + zmemcpy(s->window, s->window+wsize, (unsigned)wsize); |
| + s->match_start -= wsize; |
| + s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ |
| + s->block_start -= (long) wsize; |
| + |
| + /* Slide the hash table (could be avoided with 32 bit values |
| + at the expense of memory usage). We slide even when level == 0 |
| + to keep the hash table consistent if we switch back to level > 0 |
| + later. (Using level 0 permanently is not an optimal usage of |
| + zlib, so we don't care about this pathological case.) |
| + */ |
| + n = s->hash_size; |
| + p = &s->head[n]; |
| + p -= 8; |
| + do { |
| + __m128i value, result; |
| + |
| + value = _mm_loadu_si128((__m128i *)p); |
| + result = _mm_subs_epu16(value, xmm_wsize); |
| + _mm_storeu_si128((__m128i *)p, result); |
| + |
| + p -= 8; |
| + n -= 8; |
| + } while (n > 0); |
| + |
| + n = wsize; |
| +#ifndef FASTEST |
| + p = &s->prev[n]; |
| + p -= 8; |
| + do { |
| + __m128i value, result; |
| + |
| + value = _mm_loadu_si128((__m128i *)p); |
| + result = _mm_subs_epu16(value, xmm_wsize); |
| + _mm_storeu_si128((__m128i *)p, result); |
| + |
| + p -= 8; |
| + n -= 8; |
| + } while (n > 0); |
| +#endif |
| + more += wsize; |
| + } |
| + if (s->strm->avail_in == 0) break; |
| + |
| + /* If there was no sliding: |
| + * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && |
| + * more == window_size - lookahead - strstart |
| + * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) |
| + * => more >= window_size - 2*WSIZE + 2 |
| + * In the BIG_MEM or MMAP case (not yet supported), |
| + * window_size == input_size + MIN_LOOKAHEAD && |
| + * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. |
| + * Otherwise, window_size == 2*WSIZE so more >= 2. |
| + * If there was sliding, more >= WSIZE. So in all cases, more >= 2. |
| + */ |
| + Assert(more >= 2, "more < 2"); |
| + |
| + n = deflate_read_buf(s->strm, |
| + s->window + s->strstart + s->lookahead, |
| + more); |
| + s->lookahead += n; |
| + |
| + /* Initialize the hash value now that we have some input: */ |
| + if (s->lookahead >= MIN_MATCH) { |
| + uInt str = s->strstart; |
| + s->ins_h = s->window[str]; |
| + if (str >= 1) |
| + UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH-1)); |
| +#if MIN_MATCH != 3 |
| + Call UPDATE_HASH() MIN_MATCH-3 more times |
| +#endif |
| + } |
| + /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, |
| + * but this is not important since only literal bytes will be emitted. |
| + */ |
| + |
| + } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); |
| + |
| + /* If the WIN_INIT bytes after the end of the current data have never been |
| + * written, then zero those bytes in order to avoid memory check reports of |
| + * the use of uninitialized (or uninitialised as Julian writes) bytes by |
| + * the longest match routines. Update the high water mark for the next |
| + * time through here. WIN_INIT is set to MAX_MATCH since the longest match |
| + * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. |
| + */ |
| + if (s->high_water < s->window_size) { |
| + ulg curr = s->strstart + (ulg)(s->lookahead); |
| + ulg init; |
| + |
| + if (s->high_water < curr) { |
| + /* Previous high water mark below current data -- zero WIN_INIT |
| + * bytes or up to end of window, whichever is less. |
| + */ |
| + init = s->window_size - curr; |
| + if (init > WIN_INIT) |
| + init = WIN_INIT; |
| + zmemzero(s->window + curr, (unsigned)init); |
| + s->high_water = curr + init; |
| + } |
| + else if (s->high_water < (ulg)curr + WIN_INIT) { |
| + /* High water mark at or above current data, but below current data |
| + * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up |
| + * to end of window, whichever is less. |
| + */ |
| + init = (ulg)curr + WIN_INIT - s->high_water; |
| + if (init > s->window_size - s->high_water) |
| + init = s->window_size - s->high_water; |
| + zmemzero(s->window + s->high_water, (unsigned)init); |
| + s->high_water += init; |
| + } |
| + } |
| + |
| + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, |
| + "not enough room for search"); |
| +} |
| diff --git a/simd_stub.c b/simd_stub.c |
| new file mode 100644 |
| index 000000000000..c6d46051498f |
| --- /dev/null |
| +++ b/simd_stub.c |
| @@ -0,0 +1,35 @@ |
| +/* simd_stub.c -- stub implementations |
| +* Copyright (C) 2014 Intel Corporation |
| +* For conditions of distribution and use, see copyright notice in zlib.h |
| +*/ |
| +#include <assert.h> |
| + |
| +#include "deflate.h" |
| +#include "x86.h" |
| + |
| +int ZLIB_INTERNAL x86_cpu_enable_simd = 0; |
| + |
| +void ZLIB_INTERNAL crc_fold_init(deflate_state *const s) { |
| + assert(0); |
| +} |
| + |
| +void ZLIB_INTERNAL crc_fold_copy(deflate_state *const s, |
| + unsigned char *dst, |
| + const unsigned char *src, |
| + long len) { |
| + assert(0); |
| +} |
| + |
| +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) { |
| + assert(0); |
| + return 0; |
| +} |
| + |
| +void ZLIB_INTERNAL fill_window_sse(deflate_state *s) |
| +{ |
| + assert(0); |
| +} |
| + |
| +void x86_check_features(void) |
| +{ |
| +} |
| diff --git a/x86.c b/x86.c |
| new file mode 100644 |
| index 000000000000..e56fe8b85a39 |
| --- /dev/null |
| +++ b/x86.c |
| @@ -0,0 +1,92 @@ |
| +/* |
| + * x86 feature check |
| + * |
| + * Copyright (C) 2013 Intel Corporation. All rights reserved. |
| + * Author: |
| + * Jim Kukunas |
| + * |
| + * For conditions of distribution and use, see copyright notice in zlib.h |
| + */ |
| + |
| +#include "x86.h" |
| +#include "zutil.h" |
| + |
| +int ZLIB_INTERNAL x86_cpu_enable_simd = 0; |
| + |
| +#ifndef _MSC_VER |
| +#include <pthread.h> |
| + |
| +pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; |
| +static void _x86_check_features(void); |
| + |
| +void x86_check_features(void) |
| +{ |
| + pthread_once(&cpu_check_inited_once, _x86_check_features); |
| +} |
| + |
| +static void _x86_check_features(void) |
| +{ |
| + int x86_cpu_has_sse2; |
| + int x86_cpu_has_sse42; |
| + int x86_cpu_has_pclmulqdq; |
| + unsigned eax, ebx, ecx, edx; |
| + |
| + eax = 1; |
| +#ifdef __i386__ |
| + __asm__ __volatile__ ( |
| + "xchg %%ebx, %1\n\t" |
| + "cpuid\n\t" |
| + "xchg %1, %%ebx\n\t" |
| + : "+a" (eax), "=S" (ebx), "=c" (ecx), "=d" (edx) |
| + ); |
| +#else |
| + __asm__ __volatile__ ( |
| + "cpuid\n\t" |
| + : "+a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) |
| + ); |
| +#endif /* (__i386__) */ |
| + |
| + x86_cpu_has_sse2 = edx & 0x4000000; |
| + x86_cpu_has_sse42 = ecx & 0x100000; |
| + x86_cpu_has_pclmulqdq = ecx & 0x2; |
| + |
| + x86_cpu_enable_simd = x86_cpu_has_sse2 && |
| + x86_cpu_has_sse42 && |
| + x86_cpu_has_pclmulqdq; |
| +} |
| +#else |
| +#include <intrin.h> |
| +#include <windows.h> |
| + |
| +static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, |
| + PVOID param, |
| + PVOID *context); |
| +static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; |
| + |
| +void x86_check_features(void) |
| +{ |
| + InitOnceExecuteOnce(&cpu_check_inited_once, _x86_check_features, |
| + NULL, NULL); |
| +} |
| + |
| +static BOOL CALLBACK _x86_check_features(PINIT_ONCE once, |
| + PVOID param, |
| + PVOID *context) |
| +{ |
| + int x86_cpu_has_sse2; |
| + int x86_cpu_has_sse42; |
| + int x86_cpu_has_pclmulqdq; |
| + int regs[4]; |
| + |
| + __cpuid(regs, 1); |
| + |
| + x86_cpu_has_sse2 = regs[3] & 0x4000000; |
| + x86_cpu_has_sse42= regs[2] & 0x100000; |
| + x86_cpu_has_pclmulqdq = regs[2] & 0x2; |
| + |
| + x86_cpu_enable_simd = x86_cpu_has_sse2 && |
| + x86_cpu_has_sse42 && |
| + x86_cpu_has_pclmulqdq; |
| + return TRUE; |
| +} |
| +#endif /* _MSC_VER */ |
| diff --git a/x86.h b/x86.h |
| new file mode 100644 |
| index 000000000000..ebcf10ab09d2 |
| --- /dev/null |
| +++ b/x86.h |
| @@ -0,0 +1,15 @@ |
| +/* x86.h -- check for x86 CPU features |
| +* Copyright (C) 2013 Intel Corporation Jim Kukunas |
| +* For conditions of distribution and use, see copyright notice in zlib.h |
| +*/ |
| + |
| +#ifndef X86_H |
| +#define X86_H |
| + |
| +#include "zlib.h" |
| + |
| +extern int x86_cpu_enable_simd; |
| + |
| +void x86_check_features(void); |
| + |
| +#endif /* X86_H */ |
| diff --git a/zutil.h b/zutil.h |
| index 80375b8b6109..4425bcf75eb3 100644 |
| --- a/zutil.h |
| +++ b/zutil.h |
| @@ -283,4 +283,10 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ |
| #define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ |
| (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) |
| |
| +#ifdef _MSC_VER |
| +#define zalign(x) __declspec(align(x)) |
| +#else |
| +#define zalign(x) __attribute__((aligned((x)))) |
| +#endif |
| + |
| #endif /* ZUTIL_H */ |