[v1,3/3] x86: Optimize memcmp-evex-movbe.S

Message ID 20210517184406.2609574-3-goldstein.w.n@gmail.com
State Superseded
Headers show
Series
  • [v1,1/3] x86: Expand bench-memcmp.c and test-memcmp.c
Related show

Commit Message

Anton Blanchard via Libc-alpha May 17, 2021, 6:44 p.m.
No bug. This commit optimizes memcmp-evex.S. The optimizations include
adding a new vec compare path for small sizes, reorganizing the entry
control flow, removing some unnecissary ALU instructions from the main
loop, and most importantly replacing the heavy use of vpcmp + kand
logic with vpxor + vptern. test-memcmp and test-wmemcmp are both
passing.

Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>

---
 sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 710 +++++++++++--------
 1 file changed, 408 insertions(+), 302 deletions(-)

-- 
2.29.2

Comments

Anton Blanchard via Libc-alpha May 19, 2021, 1:03 a.m. | #1
On Mon, May 17, 2021 at 02:44:08PM -0400, Noah Goldstein wrote:
> No bug. This commit optimizes memcmp-evex.S. The optimizations include

> adding a new vec compare path for small sizes, reorganizing the entry

> control flow, removing some unnecissary ALU instructions from the main

> loop, and most importantly replacing the heavy use of vpcmp + kand

> logic with vpxor + vptern. test-memcmp and test-wmemcmp are both

> passing.

> 

> Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>

> ---

>  sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 710 +++++++++++--------

>  1 file changed, 408 insertions(+), 302 deletions(-)

> 

> diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

> index 9c093972e1..654dc7ac8c 100644

> --- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

> +++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

> @@ -19,17 +19,22 @@

>  #if IS_IN (libc)

>  

>  /* memcmp/wmemcmp is implemented as:

> -   1. For size from 2 to 7 bytes, load as big endian with movbe and bswap

> -      to avoid branches.

> -   2. Use overlapping compare to avoid branch.

> -   3. Use vector compare when size >= 4 bytes for memcmp or size >= 8

> -      bytes for wmemcmp.

> -   4. If size is 8 * VEC_SIZE or less, unroll the loop.

> -   5. Compare 4 * VEC_SIZE at a time with the aligned first memory

> +   1. Use ymm vector compares when possible. The only case where

> +      vector compares is not possible for when size < CHAR_PER_VEC

> +      and loading from either s1 or s2 would cause a page cross.

> +   2. For size from 2 to 7 bytes on page cross, load as big endian

> +      with movbe and bswap to avoid branches.

> +   3. Use xmm vector compare when size >= 4 bytes for memcmp or

> +      size >= 8 bytes for wmemcmp.

> +   4. Optimistically compare up to first 4 * CHAR_PER_VEC one at a

> +      to check for early mismatches. Only do this if its guranteed the

> +      work is not wasted.

> +   5. If size is 8 * VEC_SIZE or less, unroll the loop.

> +   6. Compare 4 * VEC_SIZE at a time with the aligned first memory

>        area.

> -   6. Use 2 vector compares when size is 2 * VEC_SIZE or less.

> -   7. Use 4 vector compares when size is 4 * VEC_SIZE or less.

> -   8. Use 8 vector compares when size is 8 * VEC_SIZE or less.  */

> +   7. Use 2 vector compares when size is 2 * CHAR_PER_VEC or less.

> +   8. Use 4 vector compares when size is 4 * CHAR_PER_VEC or less.

> +   9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less.  */

>  

>  # include <sysdep.h>

>  

> @@ -40,11 +45,21 @@

>  # define VMOVU		vmovdqu64

>  

>  # ifdef USE_AS_WMEMCMP

> -#  define VPCMPEQ	vpcmpeqd

> +#  define CHAR_SIZE	4

> +#  define VPCMP	vpcmpd

>  # else

> -#  define VPCMPEQ	vpcmpeqb

> +#  define CHAR_SIZE	1

> +#  define VPCMP	vpcmpub

>  # endif

>  

> +# define VEC_SIZE	32

> +# define PAGE_SIZE	4096

> +# define CHAR_PER_VEC	(VEC_SIZE / CHAR_SIZE)

> +

> +# define XMM0		xmm16

> +# define XMM1		xmm17

> +# define XMM2		xmm18

> +# define YMM0		ymm16

>  # define XMM1		xmm17

>  # define XMM2		xmm18

>  # define YMM1		ymm17

> @@ -54,15 +69,6 @@

>  # define YMM5		ymm21

>  # define YMM6		ymm22

>  

> -# define VEC_SIZE 32

> -# ifdef USE_AS_WMEMCMP

> -#  define VEC_MASK 0xff

> -#  define XMM_MASK 0xf

> -# else

> -#  define VEC_MASK 0xffffffff

> -#  define XMM_MASK 0xffff

> -# endif

> -

>  /* Warning!

>             wmemcmp has to use SIGNED comparison for elements.

>             memcmp has to use UNSIGNED comparison for elemnts.

> @@ -70,145 +76,370 @@

>  

>  	.section .text.evex,"ax",@progbits

>  ENTRY (MEMCMP)

> -# ifdef USE_AS_WMEMCMP

> -	shl	$2, %RDX_LP

> -# elif defined __ILP32__

> +# ifdef __ILP32__

>  	/* Clear the upper 32 bits.  */

>  	movl	%edx, %edx

>  # endif

> -	cmp	$VEC_SIZE, %RDX_LP

> +	cmp	$CHAR_PER_VEC, %RDX_LP

>  	jb	L(less_vec)

>  

>  	/* From VEC to 2 * VEC.  No branch when size == VEC_SIZE.  */

> -	VMOVU	(%rsi), %YMM2

> -	VPCMPEQ (%rdi), %YMM2, %k1

> +	VMOVU	(%rsi), %YMM1

> +	/* Use compare not equals to directly check for mismatch.  */

> +	VPCMP	$4, (%rdi), %YMM1, %k1

>  	kmovd	%k1, %eax

> -	subl    $VEC_MASK, %eax

> -	jnz	L(first_vec)

> -

> -	cmpq	$(VEC_SIZE * 2), %rdx

> -	jbe	L(last_vec)

> -

> -	/* More than 2 * VEC.  */

> -	cmpq	$(VEC_SIZE * 8), %rdx

> -	ja	L(more_8x_vec)

> -	cmpq	$(VEC_SIZE * 4), %rdx

> -	jb	L(last_4x_vec)

> +	/* NB: eax must be destination register if going to

> +	   L(return_vec_[0,2]). For L(return_vec_3 destination register

> +	   must be ecx.  */

> +	testl	%eax, %eax

> +	jnz	L(return_vec_0)

>  

> -	/* From 4 * VEC to 8 * VEC, inclusively. */

> -	VMOVU	(%rsi), %YMM1

> -	VPCMPEQ (%rdi), %YMM1, %k1

> +	cmpq	$(CHAR_PER_VEC * 2), %rdx

> +	jbe	L(last_1x_vec)

>  

> +	/* Check second VEC no matter what.  */

>  	VMOVU	VEC_SIZE(%rsi), %YMM2

> -	VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2

> +	VPCMP	$4, VEC_SIZE(%rdi), %YMM2, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_1)

> +

> +	/* Less than 4 * VEC.  */

> +	cmpq	$(CHAR_PER_VEC * 4), %rdx

> +	jbe	L(last_2x_vec)

>  

> +	/* Check third and fourth VEC no matter what.  */

>  	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM3

> -	VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3

> +	VPCMP	$4, (VEC_SIZE * 2)(%rdi), %YMM3, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_2)

>  

>  	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM4

> -	VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4

> +	VPCMP	$4, (VEC_SIZE * 3)(%rdi), %YMM4, %k1

> +	kmovd	%k1, %ecx

> +	testl	%ecx, %ecx

> +	jnz	L(return_vec_3)

>  

> -	kandd	%k1, %k2, %k5

> -	kandd	%k3, %k4, %k6

> -	kandd	%k5, %k6, %k6

> +	/* Zero YMM0. 4x VEC reduction is done with vpxor + vtern so

> +	   compare with zero to get a mask is needed.  */

> +	vpxorq	%XMM0, %XMM0, %XMM0

>  

> -	kmovd	%k6, %eax

> -	cmpl	$VEC_MASK, %eax

> -	jne	L(4x_vec_end)

> +	/* Go to 4x VEC loop.  */

> +	cmpq	$(CHAR_PER_VEC * 8), %rdx

> +	ja	L(more_8x_vec)

>  

> -	leaq	-(4 * VEC_SIZE)(%rdi, %rdx), %rdi

> -	leaq	-(4 * VEC_SIZE)(%rsi, %rdx), %rsi

> -	VMOVU	(%rsi), %YMM1

> -	VPCMPEQ (%rdi), %YMM1, %k1

> +	/* Handle remainder of size = 4 * VEC + 1 to 8 * VEC without any

> +	   branches.  */

>  

> -	VMOVU	VEC_SIZE(%rsi), %YMM2

> -	VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2

> -	kandd	%k1, %k2, %k5

> +	/* Load first two VEC from s2 before adjusting addresses.  */

> +	VMOVU	-(VEC_SIZE * 4)(%rsi, %rdx, CHAR_SIZE), %YMM1

> +	VMOVU	-(VEC_SIZE * 3)(%rsi, %rdx, CHAR_SIZE), %YMM2

> +	leaq	-(4 * VEC_SIZE)(%rdi, %rdx, CHAR_SIZE), %rdi

> +	leaq	-(4 * VEC_SIZE)(%rsi, %rdx, CHAR_SIZE), %rsi

> +

> +	/* Wait to load from s1 until addressed adjust due to

> +	   unlamination of microfusion with complex address mode.  */

> +

> +	/* vpxor will be all 0s if s1 and s2 are equal. Otherwise it

> +	   will have some 1s.  */

> +	vpxorq	(%rdi), %YMM1, %YMM1

> +	vpxorq	(VEC_SIZE)(%rdi), %YMM2, %YMM2

>  

>  	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM3

> -	VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3

> -	kandd	%k3, %k5, %k5

> +	vpxorq	(VEC_SIZE * 2)(%rdi), %YMM3, %YMM3

> +	/* Or together YMM1, YMM2, and YMM3 into YMM3.  */

> +	vpternlogd $0xfe, %YMM1, %YMM2, %YMM3

>  

>  	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM4

> -	VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4

> -	kandd	%k4, %k5, %k5

> +	/* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while

> +	   oring with YMM3. Result is stored in YMM4.  */

> +	vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4

> +	/* Compare YMM4 with 0. If any 1s s1 and s2 don't match.  */

> +	VPCMP	$4, %YMM4, %YMM0, %k1

> +	kmovd	%k1, %ecx

> +	testl	%ecx, %ecx

> +	jnz	L(return_vec_0_1_2_3)

> +	/* NB: eax must be zero to reach here.  */

> +	ret

>  

> -	kmovd	%k5, %eax

> -	cmpl	$VEC_MASK, %eax

> -	jne	L(4x_vec_end)

> -	xorl	%eax, %eax

> +	/* NB: aligning 32 here allows for the rest of the jump targets

> +	   to be tuned for 32 byte alignment. Most important this ensures

> +	   the L(more_8x_vec) loop is 32 byte aligned.  */

> +	.p2align 5

> +L(less_vec):

> +	/* Check if one or less CHAR. This is necessary for size = 0 but

> +	   is also faster for size = CHAR_SIZE.  */

> +	cmpl	$1, %edx

> +	jbe	L(one_or_less)

> +

> +	/* Check if loading one VEC from either s1 or s2 could cause a

> +	   page cross. This can have false positives but is by far the

> +	   fastest method.  */

> +	movl	%edi, %eax

> +	orl	%esi, %eax

> +	andl	$(PAGE_SIZE - 1), %eax

> +	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax

> +	jg	L(page_cross_less_vec)

> +

> +	/* No page cross possible.  */

> +	VMOVU	(%rsi), %YMM2

> +	VPCMP	$4, (%rdi), %YMM2, %k1

> +	kmovd	%k1, %eax

> +	/* Create mask in ecx for potentially in bound matches.  */

> +	bzhil	%edx, %eax, %eax

> +	jnz	L(return_vec_0)

>  	ret

>  

>  	.p2align 4

> -L(last_2x_vec):

> -	/* From VEC to 2 * VEC.  No branch when size == VEC_SIZE.  */

> -	VMOVU	(%rsi), %YMM2

> -	VPCMPEQ (%rdi), %YMM2, %k2

> -	kmovd	%k2, %eax

> -	subl    $VEC_MASK, %eax

> -	jnz	L(first_vec)

> +L(return_vec_0):

> +	tzcntl	%eax, %eax

> +# ifdef USE_AS_WMEMCMP

> +	movl	(%rdi, %rax, CHAR_SIZE), %ecx

> +	xorl	%edx, %edx

> +	cmpl	(%rsi, %rax, CHAR_SIZE), %ecx

> +	/* NB: no partial register stall here because xorl zero idiom

> +	   above.  */

> +	setg	%dl

> +	leal	-1(%rdx, %rdx), %eax

> +# else

> +	movzbl	(%rsi, %rax), %ecx

> +	movzbl	(%rdi, %rax), %eax

> +	subl	%ecx, %eax

> +# endif

> +	ret

>  

> -L(last_vec):

> -	/* Use overlapping loads to avoid branches.  */

> -	leaq	-VEC_SIZE(%rdi, %rdx), %rdi

> -	leaq	-VEC_SIZE(%rsi, %rdx), %rsi

> -	VMOVU	(%rsi), %YMM2

> -	VPCMPEQ (%rdi), %YMM2, %k2

> -	kmovd	%k2, %eax

> -	subl    $VEC_MASK, %eax

> -	jnz	L(first_vec)

> +	/* NB: No p2align necessary. Alignment  % 16 is naturally 1

> +	   which is good enough for a target not in a loop.  */

> +L(return_vec_1):

> +	tzcntl	%eax, %eax

> +# ifdef USE_AS_WMEMCMP

> +	movl	VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx

> +	xorl	%edx, %edx

> +	cmpl	VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx

> +	setg	%dl

> +	leal	-1(%rdx, %rdx), %eax

> +# else

> +	movzbl	VEC_SIZE(%rsi, %rax), %ecx

> +	movzbl	VEC_SIZE(%rdi, %rax), %eax

> +	subl	%ecx, %eax

> +# endif

>  	ret

>  

> -	.p2align 4

> -L(first_vec):

> -	/* A byte or int32 is different within 16 or 32 bytes.  */

> -	tzcntl	%eax, %ecx

> +	/* NB: No p2align necessary. Alignment  % 16 is naturally 2

> +	   which is good enough for a target not in a loop.  */

> +L(return_vec_2):

> +	tzcntl	%eax, %eax

>  # ifdef USE_AS_WMEMCMP

> -	xorl	%eax, %eax

> -	movl	(%rdi, %rcx, 4), %edx

> -	cmpl	(%rsi, %rcx, 4), %edx

> -L(wmemcmp_return):

> -	setl	%al

> -	negl	%eax

> -	orl	$1, %eax

> +	movl	(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx

> +	xorl	%edx, %edx

> +	cmpl	(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx

> +	setg	%dl

> +	leal	-1(%rdx, %rdx), %eax

>  # else

> -	movzbl	(%rdi, %rcx), %eax

> -	movzbl	(%rsi, %rcx), %edx

> -	sub	%edx, %eax

> +	movzbl	(VEC_SIZE * 2)(%rsi, %rax), %ecx

> +	movzbl	(VEC_SIZE * 2)(%rdi, %rax), %eax

> +	subl	%ecx, %eax

>  # endif

>  	ret

>  

> +	.p2align 4

> +L(8x_return_vec_0_1_2_3):

> +	/* Returning from L(more_8x_vec) requires restoring rsi.  */

> +	addq	%rdi, %rsi

> +L(return_vec_0_1_2_3):

> +	VPCMP	$4, %YMM1, %YMM0, %k0

> +	kmovd	%k0, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_0)

> +

> +	VPCMP	$4, %YMM2, %YMM0, %k0

> +	kmovd	%k0, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_1)

> +

> +	VPCMP	$4, %YMM3, %YMM0, %k0

> +	kmovd	%k0, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_2)

> +L(return_vec_3):

> +	tzcntl	%ecx, %ecx

>  # ifdef USE_AS_WMEMCMP

> +	movl	(VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax

> +	xorl	%edx, %edx

> +	cmpl	(VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax

> +	setg	%dl

> +	leal	-1(%rdx, %rdx), %eax

> +# else

> +	movzbl	(VEC_SIZE * 3)(%rdi, %rcx), %eax

> +	movzbl	(VEC_SIZE * 3)(%rsi, %rcx), %ecx

> +	subl	%ecx, %eax

> +# endif

> +	ret

> +

>  	.p2align 4

> -L(4):

> -	xorl	%eax, %eax

> -	movl	(%rdi), %edx

> -	cmpl	(%rsi), %edx

> -	jne	L(wmemcmp_return)

> +L(more_8x_vec):

> +	/* Set end of s1 in rdx.  */

> +	leaq	-(VEC_SIZE * 4)(%rdi, %rdx, CHAR_SIZE), %rdx

> +	/* rsi stores s2 - s1. This allows loop to only update one

> +	   pointer.  */

> +	subq	%rdi, %rsi

> +	/* Align s1 pointer.  */

> +	andq	$-VEC_SIZE, %rdi

> +	/* Adjust because first 4x vec where check already.  */

> +	subq	$-(VEC_SIZE * 4), %rdi

> +	.p2align 4

> +L(loop_4x_vec):

> +	VMOVU	(%rsi, %rdi), %YMM1

> +	vpxorq	(%rdi), %YMM1, %YMM1

> +

> +	VMOVU	VEC_SIZE(%rsi, %rdi), %YMM2

> +	vpxorq	VEC_SIZE(%rdi), %YMM2, %YMM2

> +

> +	VMOVU	(VEC_SIZE * 2)(%rsi, %rdi), %YMM3

> +	vpxorq	(VEC_SIZE * 2)(%rdi), %YMM3, %YMM3

> +	vpternlogd $0xfe, %YMM1, %YMM2, %YMM3

> +

> +	VMOVU	(VEC_SIZE * 3)(%rsi, %rdi), %YMM4

> +	vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4

> +	VPCMP	$4, %YMM4, %YMM0, %k1

> +	kmovd	%k1, %ecx

> +	testl	%ecx, %ecx

> +	jnz	L(8x_return_vec_0_1_2_3)

> +	subq	$-(VEC_SIZE * 4), %rdi

> +	cmpq	%rdx, %rdi

> +	jb	L(loop_4x_vec)

> +

> +	subq	%rdx, %rdi

> +	/* rdi has 4 * VEC_SIZE - remaining length.  */

> +	cmpl	$(VEC_SIZE * 3), %edi

> +	jae	L(8x_last_1x_vec)

> +	/* Load regardless of branch.  */

> +	VMOVU	(VEC_SIZE * 2)(%rsi, %rdx), %YMM3

> +	cmpl	$(VEC_SIZE * 2), %edi

> +	jae	L(8x_last_2x_vec)

> +

> +	VMOVU	(%rsi, %rdx), %YMM1

> +	vpxorq	(%rdx), %YMM1, %YMM1

> +

> +	VMOVU	VEC_SIZE(%rsi, %rdx), %YMM2

> +	vpxorq	VEC_SIZE(%rdx), %YMM2, %YMM2

> +

> +	vpxorq	(VEC_SIZE * 2)(%rdx), %YMM3, %YMM3

> +	vpternlogd $0xfe, %YMM1, %YMM2, %YMM3

> +

> +	VMOVU	(VEC_SIZE * 3)(%rsi, %rdx), %YMM4

> +	vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM3, %YMM4

> +	VPCMP	$4, %YMM4, %YMM0, %k1

> +	kmovd	%k1, %ecx

> +	/* Restore s1 pointer to rdi.  */

> +	movq	%rdx, %rdi

> +	testl	%ecx, %ecx

> +	jnz	L(8x_return_vec_0_1_2_3)

> +	/* NB: eax must be zero to reach here.  */

> +	ret

> +

> +	/* Only entry is from L(more_8x_vec).  */

> +	.p2align 4

> +L(8x_last_2x_vec):

> +	VPCMP	$4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(8x_return_vec_2)

> +	/* Naturally aligned to 16 bytes.  */

> +L(8x_last_1x_vec):

> +	VMOVU	(VEC_SIZE * 3)(%rsi, %rdx), %YMM1

> +	VPCMP	$4, (VEC_SIZE * 3)(%rdx), %YMM1, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(8x_return_vec_3)

> +	ret

> +

> +	.p2align 4

> +L(last_2x_vec):

> +	/* Check second to last VEC.  */

> +	VMOVU	-(VEC_SIZE * 2)(%rsi, %rdx, CHAR_SIZE), %YMM1

> +	VPCMP	$4, -(VEC_SIZE * 2)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_1_end)

> +

> +	/* Check last VEC.  */

> +	.p2align 4

> +L(last_1x_vec):

> +	VMOVU	-(VEC_SIZE * 1)(%rsi, %rdx, CHAR_SIZE), %YMM1

> +	VPCMP	$4, -(VEC_SIZE * 1)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_0_end)

>  	ret

> +

> +	.p2align 4

> +L(8x_return_vec_2):

> +	subq	$VEC_SIZE, %rdx

> +L(8x_return_vec_3):

> +	tzcntl	%eax, %eax

> +# ifdef USE_AS_WMEMCMP

> +	leaq	(%rdx, %rax, CHAR_SIZE), %rax

> +	movl	(VEC_SIZE * 3)(%rax), %ecx

> +	xorl	%edx, %edx

> +	cmpl	(VEC_SIZE * 3)(%rsi, %rax), %ecx

> +	setg	%dl

> +	leal	-1(%rdx, %rdx), %eax

>  # else

> +	addq	%rdx, %rax

> +	movzbl	(VEC_SIZE * 3)(%rsi, %rax), %ecx

> +	movzbl	(VEC_SIZE * 3)(%rax), %eax

> +	subl	%ecx, %eax

> +# endif

> +	ret

> +

>  	.p2align 4

> -L(between_4_7):

> -	/* Load as big endian with overlapping movbe to avoid branches.  */

> -	movbe	(%rdi), %eax

> -	movbe	(%rsi), %ecx

> -	shlq	$32, %rax

> -	shlq	$32, %rcx

> -	movbe	-4(%rdi, %rdx), %edi

> -	movbe	-4(%rsi, %rdx), %esi

> -	orq	%rdi, %rax

> -	orq	%rsi, %rcx

> -	subq	%rcx, %rax

> -	je	L(exit)

> -	sbbl	%eax, %eax

> -	orl	$1, %eax

> +L(return_vec_0_end):

> +	tzcntl	%eax, %eax

> +	addl	%edx, %eax

> +# ifdef USE_AS_WMEMCMP

> +	movl	-VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx

> +	xorl	%edx, %edx

> +	cmpl	-VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx

> +	setg	%dl

> +	leal	-1(%rdx, %rdx), %eax

> +# else

> +	movzbl	-VEC_SIZE(%rsi, %rax), %ecx

> +	movzbl	-VEC_SIZE(%rdi, %rax), %eax

> +	subl	%ecx, %eax

> +# endif

>  	ret

>  

>  	.p2align 4

> -L(exit):

> +L(return_vec_1_end):

> +	tzcntl	%eax, %eax

> +	addl	%edx, %eax

> +# ifdef USE_AS_WMEMCMP

> +	movl	-(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx

> +	xorl	%edx, %edx

> +	cmpl	-(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx

> +	setg	%dl

> +	leal	-1(%rdx, %rdx), %eax

> +# else

> +	movzbl	-(VEC_SIZE * 2)(%rsi, %rax), %ecx

> +	movzbl	-(VEC_SIZE * 2)(%rdi, %rax), %eax

> +	subl	%ecx, %eax

> +# endif

>  	ret

>  

> +

>  	.p2align 4

> +L(page_cross_less_vec):

> +	/* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28

> +	   bytes.  */

> +	cmpl	$(16 / CHAR_SIZE), %edx

> +	jae	L(between_16_31)

> +# ifndef USE_AS_WMEMCMP

> +	cmpl	$8, %edx

> +	jae	L(between_8_15)

> +	cmpl	$4, %edx

> +	jae	L(between_4_7)

>  L(between_2_3):

>  	/* Load as big endian to avoid branches.  */

>  	movzwl	(%rdi), %eax

> @@ -217,224 +448,99 @@ L(between_2_3):

>  	shll	$8, %ecx

>  	bswap	%eax

>  	bswap	%ecx

> -	movb	-1(%rdi, %rdx), %al

> -	movb	-1(%rsi, %rdx), %cl

> +	movzbl	-1(%rdi, %rdx), %edi

> +	movzbl	-1(%rsi, %rdx), %esi

> +	orl	%edi, %eax

> +	orl	%esi, %ecx

>  	/* Subtraction is okay because the upper 8 bits are zero.  */

>  	subl	%ecx, %eax

>  	ret

> -

>  	.p2align 4

> -L(1):

> -	movzbl	(%rdi), %eax

> +L(one_or_less):

> +	jb	L(zero)

>  	movzbl	(%rsi), %ecx

> +	movzbl	(%rdi), %eax

>  	subl	%ecx, %eax

>  	ret

> -# endif

> -

> -	.p2align 4

> -L(zero):

> -	xorl	%eax, %eax

> -	ret

>  

>  	.p2align 4

> -L(less_vec):

> -# ifdef USE_AS_WMEMCMP

> -	/* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes.  */

> -	cmpb	$4, %dl

> -	je	L(4)

> -	jb	L(zero)

> -# else

> -	cmpb	$1, %dl

> -	je	L(1)

> -	jb	L(zero)

> -	cmpb	$4, %dl

> -	jb	L(between_2_3)

> -	cmpb	$8, %dl

> -	jb	L(between_4_7)

> +L(between_8_15):

>  # endif

> -	cmpb	$16, %dl

> -	jae	L(between_16_31)

> -	/* It is between 8 and 15 bytes.  */

> +	/* If USE_AS_WMEMCMP fall through into 8-15 byte case.  */

>  	vmovq	(%rdi), %XMM1

>  	vmovq	(%rsi), %XMM2

> -	VPCMPEQ %XMM1, %XMM2, %k2

> -	kmovw	%k2, %eax

> -	subl    $XMM_MASK, %eax

> -	jnz	L(first_vec)

> +	VPCMP	$4, %XMM1, %XMM2, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_0)

>  	/* Use overlapping loads to avoid branches.  */

> -	leaq	-8(%rdi, %rdx), %rdi

> -	leaq	-8(%rsi, %rdx), %rsi

> +	leaq	-8(%rdi, %rdx, CHAR_SIZE), %rdi

> +	leaq	-8(%rsi, %rdx, CHAR_SIZE), %rsi

>  	vmovq	(%rdi), %XMM1

>  	vmovq	(%rsi), %XMM2

> -	VPCMPEQ %XMM1, %XMM2, %k2

> -	kmovw	%k2, %eax

> -	subl    $XMM_MASK, %eax

> -	jnz	L(first_vec)

> +	VPCMP	$4, %XMM1, %XMM2, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_0)

>  	ret

>  

>  	.p2align 4

> -L(between_16_31):

> -	/* From 16 to 31 bytes.  No branch when size == 16.  */

> -	VMOVU	(%rsi), %XMM2

> -	VPCMPEQ (%rdi), %XMM2, %k2

> -	kmovw	%k2, %eax

> -	subl    $XMM_MASK, %eax

> -	jnz	L(first_vec)

> -

> -	/* Use overlapping loads to avoid branches.  */

> -	leaq	-16(%rdi, %rdx), %rdi

> -	leaq	-16(%rsi, %rdx), %rsi

> -	VMOVU	(%rsi), %XMM2

> -	VPCMPEQ (%rdi), %XMM2, %k2

> -	kmovw	%k2, %eax

> -	subl    $XMM_MASK, %eax

> -	jnz	L(first_vec)

> +L(zero):

> +	xorl	%eax, %eax

>  	ret

>  

>  	.p2align 4

> -L(more_8x_vec):

> -	/* More than 8 * VEC.  Check the first VEC.  */

> -	VMOVU	(%rsi), %YMM2

> -	VPCMPEQ (%rdi), %YMM2, %k2

> -	kmovd	%k2, %eax

> -	subl    $VEC_MASK, %eax

> -	jnz	L(first_vec)

> -

> -	/* Align the first memory area for aligned loads in the loop.

> -	   Compute how much the first memory area is misaligned.  */

> -	movq	%rdi, %rcx

> -	andl	$(VEC_SIZE - 1), %ecx

> -	/* Get the negative of offset for alignment.  */

> -	subq	$VEC_SIZE, %rcx

> -	/* Adjust the second memory area.  */

> -	subq	%rcx, %rsi

> -	/* Adjust the first memory area which should be aligned now.  */

> -	subq	%rcx, %rdi

> -	/* Adjust length.  */

> -	addq	%rcx, %rdx

> -

> -L(loop_4x_vec):

> -	/* Compare 4 * VEC at a time forward.  */

> -	VMOVU	(%rsi), %YMM1

> -	VPCMPEQ (%rdi), %YMM1, %k1

> -

> -	VMOVU	VEC_SIZE(%rsi), %YMM2

> -	VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2

> -	kandd	%k2, %k1, %k5

> -

> -	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM3

> -	VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3

> -	kandd	%k3, %k5, %k5

> -

> -	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM4

> -	VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4

> -	kandd	%k4, %k5, %k5

> -

> -	kmovd	%k5, %eax

> -	cmpl	$VEC_MASK, %eax

> -	jne	L(4x_vec_end)

> -

> -	addq	$(VEC_SIZE * 4), %rdi

> -	addq	$(VEC_SIZE * 4), %rsi

> -

> -	subq	$(VEC_SIZE * 4), %rdx

> -	cmpq	$(VEC_SIZE * 4), %rdx

> -	jae	L(loop_4x_vec)

> -

> -	/* Less than 4 * VEC.  */

> -	cmpq	$VEC_SIZE, %rdx

> -	jbe	L(last_vec)

> -	cmpq	$(VEC_SIZE * 2), %rdx

> -	jbe	L(last_2x_vec)

> -

> -L(last_4x_vec):

> -	/* From 2 * VEC to 4 * VEC. */

> -	VMOVU	(%rsi), %YMM2

> -	VPCMPEQ (%rdi), %YMM2, %k2

> -	kmovd	%k2, %eax

> -	subl    $VEC_MASK, %eax

> -	jnz	L(first_vec)

> -

> -	addq	$VEC_SIZE, %rdi

> -	addq	$VEC_SIZE, %rsi

> -	VMOVU	(%rsi), %YMM2

> -	VPCMPEQ (%rdi), %YMM2, %k2

> -	kmovd	%k2, %eax

> -	subl    $VEC_MASK, %eax

> -	jnz	L(first_vec)

> +L(between_16_31):

> +	/* From 16 to 31 bytes.  No branch when size == 16.  */

> +	VMOVU	(%rsi), %XMM2

> +	VPCMP	$4, (%rdi), %XMM2, %k1

> +	kmovd	%k1, %eax

> +	testl	%eax, %eax

> +	jnz	L(return_vec_0)

>  

>  	/* Use overlapping loads to avoid branches.  */

> -	leaq	-(3 * VEC_SIZE)(%rdi, %rdx), %rdi

> -	leaq	-(3 * VEC_SIZE)(%rsi, %rdx), %rsi

> -	VMOVU	(%rsi), %YMM2

> -	VPCMPEQ (%rdi), %YMM2, %k2

> -	kmovd	%k2, %eax

> -	subl    $VEC_MASK, %eax

> -	jnz	L(first_vec)

>  

> -	addq	$VEC_SIZE, %rdi

> -	addq	$VEC_SIZE, %rsi

> -	VMOVU	(%rsi), %YMM2

> -	VPCMPEQ (%rdi), %YMM2, %k2

> -	kmovd	%k2, %eax

> -	subl    $VEC_MASK, %eax

> -	jnz	L(first_vec)

> -	ret

> -

> -	.p2align 4

> -L(4x_vec_end):

> +	VMOVU	-16(%rsi, %rdx, CHAR_SIZE), %XMM2

> +	leaq	-16(%rdi, %rdx, CHAR_SIZE), %rdi

> +	leaq	-16(%rsi, %rdx, CHAR_SIZE), %rsi

> +	VPCMP	$4, (%rdi), %XMM2, %k1

>  	kmovd	%k1, %eax

> -	subl	$VEC_MASK, %eax

> -	jnz	L(first_vec)

> -	kmovd	%k2, %eax

> -	subl	$VEC_MASK, %eax

> -	jnz	L(first_vec_x1)

> -	kmovd	%k3, %eax

> -	subl	$VEC_MASK, %eax

> -	jnz	L(first_vec_x2)

> -	kmovd	%k4, %eax

> -	subl	$VEC_MASK, %eax

> -	tzcntl	%eax, %ecx

> -# ifdef USE_AS_WMEMCMP

> -	xorl	%eax, %eax

> -	movl	(VEC_SIZE * 3)(%rdi, %rcx, 4), %edx

> -	cmpl	(VEC_SIZE * 3)(%rsi, %rcx, 4), %edx

> -	jmp	L(wmemcmp_return)

> -# else

> -	movzbl	(VEC_SIZE * 3)(%rdi, %rcx), %eax

> -	movzbl	(VEC_SIZE * 3)(%rsi, %rcx), %edx

> -	sub	%edx, %eax

> -# endif

> +	testl	%eax, %eax

> +	jnz	L(return_vec_0)

>  	ret

>  

> -	.p2align 4

> -L(first_vec_x1):

> -	tzcntl	%eax, %ecx

>  # ifdef USE_AS_WMEMCMP

> -	xorl	%eax, %eax

> -	movl	VEC_SIZE(%rdi, %rcx, 4), %edx

> -	cmpl	VEC_SIZE(%rsi, %rcx, 4), %edx

> -	jmp	L(wmemcmp_return)

> -# else

> -	movzbl	VEC_SIZE(%rdi, %rcx), %eax

> -	movzbl	VEC_SIZE(%rsi, %rcx), %edx

> -	sub	%edx, %eax

> -# endif

> +	.p2align 4

> +L(one_or_less):

> +	jb	L(zero)

> +	movl	(%rdi), %ecx

> +	xorl	%edx, %edx

> +	cmpl	(%rsi), %ecx

> +	je	L(zero)

> +	setg	%dl

> +	leal	-1(%rdx, %rdx), %eax

>  	ret

> +# else

>  

>  	.p2align 4

> -L(first_vec_x2):

> -	tzcntl	%eax, %ecx

> -# ifdef USE_AS_WMEMCMP

> -	xorl	%eax, %eax

> -	movl	(VEC_SIZE * 2)(%rdi, %rcx, 4), %edx

> -	cmpl	(VEC_SIZE * 2)(%rsi, %rcx, 4), %edx

> -	jmp	L(wmemcmp_return)

> -# else

> -	movzbl	(VEC_SIZE * 2)(%rdi, %rcx), %eax

> -	movzbl	(VEC_SIZE * 2)(%rsi, %rcx), %edx

> -	sub	%edx, %eax

> -# endif

> +L(between_4_7):

> +	/* Load as big endian with overlapping movbe to avoid branches.

> +	 */

> +	movbe	(%rdi), %eax

> +	movbe	(%rsi), %ecx

> +	shlq	$32, %rax

> +	shlq	$32, %rcx

> +	movbe	-4(%rdi, %rdx), %edi

> +	movbe	-4(%rsi, %rdx), %esi

> +	orq	%rdi, %rax

> +	orq	%rsi, %rcx

> +	subq	%rcx, %rax

> +	jz	L(zero_4_7)

> +	sbbl	%eax, %eax

> +	orl	$1, %eax

> +L(zero_4_7):

>  	ret

> +# endif

> +

>  END (MEMCMP)

>  #endif

> -- 

> 2.29.2

> 


OK, pending ifunc-impl-list.c change to require BMI2 for AVX2/EVEX
memcmp.

Thanks.

H.J.
Anton Blanchard via Libc-alpha May 19, 2021, 2:25 a.m. | #2
On Tue, May 18, 2021 at 9:03 PM H.J. Lu <hjl.tools@gmail.com> wrote:
>

> On Mon, May 17, 2021 at 02:44:08PM -0400, Noah Goldstein wrote:

> > No bug. This commit optimizes memcmp-evex.S. The optimizations include

> > adding a new vec compare path for small sizes, reorganizing the entry

> > control flow, removing some unnecissary ALU instructions from the main

> > loop, and most importantly replacing the heavy use of vpcmp + kand

> > logic with vpxor + vptern. test-memcmp and test-wmemcmp are both

> > passing.

> >

> > Signed-off-by: Noah Goldstein <goldstein.w.n@gmail.com>

> > ---

> >  sysdeps/x86_64/multiarch/memcmp-evex-movbe.S | 710 +++++++++++--------

> >  1 file changed, 408 insertions(+), 302 deletions(-)

> >

> > diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

> > index 9c093972e1..654dc7ac8c 100644

> > --- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

> > +++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S

> > @@ -19,17 +19,22 @@

> >  #if IS_IN (libc)

> >

> >  /* memcmp/wmemcmp is implemented as:

> > -   1. For size from 2 to 7 bytes, load as big endian with movbe and bswap

> > -      to avoid branches.

> > -   2. Use overlapping compare to avoid branch.

> > -   3. Use vector compare when size >= 4 bytes for memcmp or size >= 8

> > -      bytes for wmemcmp.

> > -   4. If size is 8 * VEC_SIZE or less, unroll the loop.

> > -   5. Compare 4 * VEC_SIZE at a time with the aligned first memory

> > +   1. Use ymm vector compares when possible. The only case where

> > +      vector compares is not possible for when size < CHAR_PER_VEC

> > +      and loading from either s1 or s2 would cause a page cross.

> > +   2. For size from 2 to 7 bytes on page cross, load as big endian

> > +      with movbe and bswap to avoid branches.

> > +   3. Use xmm vector compare when size >= 4 bytes for memcmp or

> > +      size >= 8 bytes for wmemcmp.

> > +   4. Optimistically compare up to first 4 * CHAR_PER_VEC one at a

> > +      to check for early mismatches. Only do this if its guranteed the

> > +      work is not wasted.

> > +   5. If size is 8 * VEC_SIZE or less, unroll the loop.

> > +   6. Compare 4 * VEC_SIZE at a time with the aligned first memory

> >        area.

> > -   6. Use 2 vector compares when size is 2 * VEC_SIZE or less.

> > -   7. Use 4 vector compares when size is 4 * VEC_SIZE or less.

> > -   8. Use 8 vector compares when size is 8 * VEC_SIZE or less.  */

> > +   7. Use 2 vector compares when size is 2 * CHAR_PER_VEC or less.

> > +   8. Use 4 vector compares when size is 4 * CHAR_PER_VEC or less.

> > +   9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less.  */

> >

> >  # include <sysdep.h>

> >

> > @@ -40,11 +45,21 @@

> >  # define VMOVU               vmovdqu64

> >

> >  # ifdef USE_AS_WMEMCMP

> > -#  define VPCMPEQ    vpcmpeqd

> > +#  define CHAR_SIZE  4

> > +#  define VPCMP      vpcmpd

> >  # else

> > -#  define VPCMPEQ    vpcmpeqb

> > +#  define CHAR_SIZE  1

> > +#  define VPCMP      vpcmpub

> >  # endif

> >

> > +# define VEC_SIZE    32

> > +# define PAGE_SIZE   4096

> > +# define CHAR_PER_VEC        (VEC_SIZE / CHAR_SIZE)

> > +

> > +# define XMM0                xmm16

> > +# define XMM1                xmm17

> > +# define XMM2                xmm18

> > +# define YMM0                ymm16

> >  # define XMM1                xmm17

> >  # define XMM2                xmm18

> >  # define YMM1                ymm17

> > @@ -54,15 +69,6 @@

> >  # define YMM5                ymm21

> >  # define YMM6                ymm22

> >

> > -# define VEC_SIZE 32

> > -# ifdef USE_AS_WMEMCMP

> > -#  define VEC_MASK 0xff

> > -#  define XMM_MASK 0xf

> > -# else

> > -#  define VEC_MASK 0xffffffff

> > -#  define XMM_MASK 0xffff

> > -# endif

> > -

> >  /* Warning!

> >             wmemcmp has to use SIGNED comparison for elements.

> >             memcmp has to use UNSIGNED comparison for elemnts.

> > @@ -70,145 +76,370 @@

> >

> >       .section .text.evex,"ax",@progbits

> >  ENTRY (MEMCMP)

> > -# ifdef USE_AS_WMEMCMP

> > -     shl     $2, %RDX_LP

> > -# elif defined __ILP32__

> > +# ifdef __ILP32__

> >       /* Clear the upper 32 bits.  */

> >       movl    %edx, %edx

> >  # endif

> > -     cmp     $VEC_SIZE, %RDX_LP

> > +     cmp     $CHAR_PER_VEC, %RDX_LP

> >       jb      L(less_vec)

> >

> >       /* From VEC to 2 * VEC.  No branch when size == VEC_SIZE.  */

> > -     VMOVU   (%rsi), %YMM2

> > -     VPCMPEQ (%rdi), %YMM2, %k1

> > +     VMOVU   (%rsi), %YMM1

> > +     /* Use compare not equals to directly check for mismatch.  */

> > +     VPCMP   $4, (%rdi), %YMM1, %k1

> >       kmovd   %k1, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> > -

> > -     cmpq    $(VEC_SIZE * 2), %rdx

> > -     jbe     L(last_vec)

> > -

> > -     /* More than 2 * VEC.  */

> > -     cmpq    $(VEC_SIZE * 8), %rdx

> > -     ja      L(more_8x_vec)

> > -     cmpq    $(VEC_SIZE * 4), %rdx

> > -     jb      L(last_4x_vec)

> > +     /* NB: eax must be destination register if going to

> > +        L(return_vec_[0,2]). For L(return_vec_3 destination register

> > +        must be ecx.  */

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_0)

> >

> > -     /* From 4 * VEC to 8 * VEC, inclusively. */

> > -     VMOVU   (%rsi), %YMM1

> > -     VPCMPEQ (%rdi), %YMM1, %k1

> > +     cmpq    $(CHAR_PER_VEC * 2), %rdx

> > +     jbe     L(last_1x_vec)

> >

> > +     /* Check second VEC no matter what.  */

> >       VMOVU   VEC_SIZE(%rsi), %YMM2

> > -     VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2

> > +     VPCMP   $4, VEC_SIZE(%rdi), %YMM2, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_1)

> > +

> > +     /* Less than 4 * VEC.  */

> > +     cmpq    $(CHAR_PER_VEC * 4), %rdx

> > +     jbe     L(last_2x_vec)

> >

> > +     /* Check third and fourth VEC no matter what.  */

> >       VMOVU   (VEC_SIZE * 2)(%rsi), %YMM3

> > -     VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3

> > +     VPCMP   $4, (VEC_SIZE * 2)(%rdi), %YMM3, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_2)

> >

> >       VMOVU   (VEC_SIZE * 3)(%rsi), %YMM4

> > -     VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4

> > +     VPCMP   $4, (VEC_SIZE * 3)(%rdi), %YMM4, %k1

> > +     kmovd   %k1, %ecx

> > +     testl   %ecx, %ecx

> > +     jnz     L(return_vec_3)

> >

> > -     kandd   %k1, %k2, %k5

> > -     kandd   %k3, %k4, %k6

> > -     kandd   %k5, %k6, %k6

> > +     /* Zero YMM0. 4x VEC reduction is done with vpxor + vtern so

> > +        compare with zero to get a mask is needed.  */

> > +     vpxorq  %XMM0, %XMM0, %XMM0

> >

> > -     kmovd   %k6, %eax

> > -     cmpl    $VEC_MASK, %eax

> > -     jne     L(4x_vec_end)

> > +     /* Go to 4x VEC loop.  */

> > +     cmpq    $(CHAR_PER_VEC * 8), %rdx

> > +     ja      L(more_8x_vec)

> >

> > -     leaq    -(4 * VEC_SIZE)(%rdi, %rdx), %rdi

> > -     leaq    -(4 * VEC_SIZE)(%rsi, %rdx), %rsi

> > -     VMOVU   (%rsi), %YMM1

> > -     VPCMPEQ (%rdi), %YMM1, %k1

> > +     /* Handle remainder of size = 4 * VEC + 1 to 8 * VEC without any

> > +        branches.  */

> >

> > -     VMOVU   VEC_SIZE(%rsi), %YMM2

> > -     VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2

> > -     kandd   %k1, %k2, %k5

> > +     /* Load first two VEC from s2 before adjusting addresses.  */

> > +     VMOVU   -(VEC_SIZE * 4)(%rsi, %rdx, CHAR_SIZE), %YMM1

> > +     VMOVU   -(VEC_SIZE * 3)(%rsi, %rdx, CHAR_SIZE), %YMM2

> > +     leaq    -(4 * VEC_SIZE)(%rdi, %rdx, CHAR_SIZE), %rdi

> > +     leaq    -(4 * VEC_SIZE)(%rsi, %rdx, CHAR_SIZE), %rsi

> > +

> > +     /* Wait to load from s1 until addressed adjust due to

> > +        unlamination of microfusion with complex address mode.  */

> > +

> > +     /* vpxor will be all 0s if s1 and s2 are equal. Otherwise it

> > +        will have some 1s.  */

> > +     vpxorq  (%rdi), %YMM1, %YMM1

> > +     vpxorq  (VEC_SIZE)(%rdi), %YMM2, %YMM2

> >

> >       VMOVU   (VEC_SIZE * 2)(%rsi), %YMM3

> > -     VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3

> > -     kandd   %k3, %k5, %k5

> > +     vpxorq  (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3

> > +     /* Or together YMM1, YMM2, and YMM3 into YMM3.  */

> > +     vpternlogd $0xfe, %YMM1, %YMM2, %YMM3

> >

> >       VMOVU   (VEC_SIZE * 3)(%rsi), %YMM4

> > -     VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4

> > -     kandd   %k4, %k5, %k5

> > +     /* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while

> > +        oring with YMM3. Result is stored in YMM4.  */

> > +     vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4

> > +     /* Compare YMM4 with 0. If any 1s s1 and s2 don't match.  */

> > +     VPCMP   $4, %YMM4, %YMM0, %k1

> > +     kmovd   %k1, %ecx

> > +     testl   %ecx, %ecx

> > +     jnz     L(return_vec_0_1_2_3)

> > +     /* NB: eax must be zero to reach here.  */

> > +     ret

> >

> > -     kmovd   %k5, %eax

> > -     cmpl    $VEC_MASK, %eax

> > -     jne     L(4x_vec_end)

> > -     xorl    %eax, %eax

> > +     /* NB: aligning 32 here allows for the rest of the jump targets

> > +        to be tuned for 32 byte alignment. Most important this ensures

> > +        the L(more_8x_vec) loop is 32 byte aligned.  */

> > +     .p2align 5

> > +L(less_vec):

> > +     /* Check if one or less CHAR. This is necessary for size = 0 but

> > +        is also faster for size = CHAR_SIZE.  */

> > +     cmpl    $1, %edx

> > +     jbe     L(one_or_less)

> > +

> > +     /* Check if loading one VEC from either s1 or s2 could cause a

> > +        page cross. This can have false positives but is by far the

> > +        fastest method.  */

> > +     movl    %edi, %eax

> > +     orl     %esi, %eax

> > +     andl    $(PAGE_SIZE - 1), %eax

> > +     cmpl    $(PAGE_SIZE - VEC_SIZE), %eax

> > +     jg      L(page_cross_less_vec)

> > +

> > +     /* No page cross possible.  */

> > +     VMOVU   (%rsi), %YMM2

> > +     VPCMP   $4, (%rdi), %YMM2, %k1

> > +     kmovd   %k1, %eax

> > +     /* Create mask in ecx for potentially in bound matches.  */

> > +     bzhil   %edx, %eax, %eax

> > +     jnz     L(return_vec_0)

> >       ret

> >

> >       .p2align 4

> > -L(last_2x_vec):

> > -     /* From VEC to 2 * VEC.  No branch when size == VEC_SIZE.  */

> > -     VMOVU   (%rsi), %YMM2

> > -     VPCMPEQ (%rdi), %YMM2, %k2

> > -     kmovd   %k2, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> > +L(return_vec_0):

> > +     tzcntl  %eax, %eax

> > +# ifdef USE_AS_WMEMCMP

> > +     movl    (%rdi, %rax, CHAR_SIZE), %ecx

> > +     xorl    %edx, %edx

> > +     cmpl    (%rsi, %rax, CHAR_SIZE), %ecx

> > +     /* NB: no partial register stall here because xorl zero idiom

> > +        above.  */

> > +     setg    %dl

> > +     leal    -1(%rdx, %rdx), %eax

> > +# else

> > +     movzbl  (%rsi, %rax), %ecx

> > +     movzbl  (%rdi, %rax), %eax

> > +     subl    %ecx, %eax

> > +# endif

> > +     ret

> >

> > -L(last_vec):

> > -     /* Use overlapping loads to avoid branches.  */

> > -     leaq    -VEC_SIZE(%rdi, %rdx), %rdi

> > -     leaq    -VEC_SIZE(%rsi, %rdx), %rsi

> > -     VMOVU   (%rsi), %YMM2

> > -     VPCMPEQ (%rdi), %YMM2, %k2

> > -     kmovd   %k2, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> > +     /* NB: No p2align necessary. Alignment  % 16 is naturally 1

> > +        which is good enough for a target not in a loop.  */

> > +L(return_vec_1):

> > +     tzcntl  %eax, %eax

> > +# ifdef USE_AS_WMEMCMP

> > +     movl    VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx

> > +     xorl    %edx, %edx

> > +     cmpl    VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx

> > +     setg    %dl

> > +     leal    -1(%rdx, %rdx), %eax

> > +# else

> > +     movzbl  VEC_SIZE(%rsi, %rax), %ecx

> > +     movzbl  VEC_SIZE(%rdi, %rax), %eax

> > +     subl    %ecx, %eax

> > +# endif

> >       ret

> >

> > -     .p2align 4

> > -L(first_vec):

> > -     /* A byte or int32 is different within 16 or 32 bytes.  */

> > -     tzcntl  %eax, %ecx

> > +     /* NB: No p2align necessary. Alignment  % 16 is naturally 2

> > +        which is good enough for a target not in a loop.  */

> > +L(return_vec_2):

> > +     tzcntl  %eax, %eax

> >  # ifdef USE_AS_WMEMCMP

> > -     xorl    %eax, %eax

> > -     movl    (%rdi, %rcx, 4), %edx

> > -     cmpl    (%rsi, %rcx, 4), %edx

> > -L(wmemcmp_return):

> > -     setl    %al

> > -     negl    %eax

> > -     orl     $1, %eax

> > +     movl    (VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx

> > +     xorl    %edx, %edx

> > +     cmpl    (VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx

> > +     setg    %dl

> > +     leal    -1(%rdx, %rdx), %eax

> >  # else

> > -     movzbl  (%rdi, %rcx), %eax

> > -     movzbl  (%rsi, %rcx), %edx

> > -     sub     %edx, %eax

> > +     movzbl  (VEC_SIZE * 2)(%rsi, %rax), %ecx

> > +     movzbl  (VEC_SIZE * 2)(%rdi, %rax), %eax

> > +     subl    %ecx, %eax

> >  # endif

> >       ret

> >

> > +     .p2align 4

> > +L(8x_return_vec_0_1_2_3):

> > +     /* Returning from L(more_8x_vec) requires restoring rsi.  */

> > +     addq    %rdi, %rsi

> > +L(return_vec_0_1_2_3):

> > +     VPCMP   $4, %YMM1, %YMM0, %k0

> > +     kmovd   %k0, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_0)

> > +

> > +     VPCMP   $4, %YMM2, %YMM0, %k0

> > +     kmovd   %k0, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_1)

> > +

> > +     VPCMP   $4, %YMM3, %YMM0, %k0

> > +     kmovd   %k0, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_2)

> > +L(return_vec_3):

> > +     tzcntl  %ecx, %ecx

> >  # ifdef USE_AS_WMEMCMP

> > +     movl    (VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax

> > +     xorl    %edx, %edx

> > +     cmpl    (VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax

> > +     setg    %dl

> > +     leal    -1(%rdx, %rdx), %eax

> > +# else

> > +     movzbl  (VEC_SIZE * 3)(%rdi, %rcx), %eax

> > +     movzbl  (VEC_SIZE * 3)(%rsi, %rcx), %ecx

> > +     subl    %ecx, %eax

> > +# endif

> > +     ret

> > +

> >       .p2align 4

> > -L(4):

> > -     xorl    %eax, %eax

> > -     movl    (%rdi), %edx

> > -     cmpl    (%rsi), %edx

> > -     jne     L(wmemcmp_return)

> > +L(more_8x_vec):

> > +     /* Set end of s1 in rdx.  */

> > +     leaq    -(VEC_SIZE * 4)(%rdi, %rdx, CHAR_SIZE), %rdx

> > +     /* rsi stores s2 - s1. This allows loop to only update one

> > +        pointer.  */

> > +     subq    %rdi, %rsi

> > +     /* Align s1 pointer.  */

> > +     andq    $-VEC_SIZE, %rdi

> > +     /* Adjust because first 4x vec where check already.  */

> > +     subq    $-(VEC_SIZE * 4), %rdi

> > +     .p2align 4

> > +L(loop_4x_vec):

> > +     VMOVU   (%rsi, %rdi), %YMM1

> > +     vpxorq  (%rdi), %YMM1, %YMM1

> > +

> > +     VMOVU   VEC_SIZE(%rsi, %rdi), %YMM2

> > +     vpxorq  VEC_SIZE(%rdi), %YMM2, %YMM2

> > +

> > +     VMOVU   (VEC_SIZE * 2)(%rsi, %rdi), %YMM3

> > +     vpxorq  (VEC_SIZE * 2)(%rdi), %YMM3, %YMM3

> > +     vpternlogd $0xfe, %YMM1, %YMM2, %YMM3

> > +

> > +     VMOVU   (VEC_SIZE * 3)(%rsi, %rdi), %YMM4

> > +     vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4

> > +     VPCMP   $4, %YMM4, %YMM0, %k1

> > +     kmovd   %k1, %ecx

> > +     testl   %ecx, %ecx

> > +     jnz     L(8x_return_vec_0_1_2_3)

> > +     subq    $-(VEC_SIZE * 4), %rdi

> > +     cmpq    %rdx, %rdi

> > +     jb      L(loop_4x_vec)

> > +

> > +     subq    %rdx, %rdi

> > +     /* rdi has 4 * VEC_SIZE - remaining length.  */

> > +     cmpl    $(VEC_SIZE * 3), %edi

> > +     jae     L(8x_last_1x_vec)

> > +     /* Load regardless of branch.  */

> > +     VMOVU   (VEC_SIZE * 2)(%rsi, %rdx), %YMM3

> > +     cmpl    $(VEC_SIZE * 2), %edi

> > +     jae     L(8x_last_2x_vec)

> > +

> > +     VMOVU   (%rsi, %rdx), %YMM1

> > +     vpxorq  (%rdx), %YMM1, %YMM1

> > +

> > +     VMOVU   VEC_SIZE(%rsi, %rdx), %YMM2

> > +     vpxorq  VEC_SIZE(%rdx), %YMM2, %YMM2

> > +

> > +     vpxorq  (VEC_SIZE * 2)(%rdx), %YMM3, %YMM3

> > +     vpternlogd $0xfe, %YMM1, %YMM2, %YMM3

> > +

> > +     VMOVU   (VEC_SIZE * 3)(%rsi, %rdx), %YMM4

> > +     vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM3, %YMM4

> > +     VPCMP   $4, %YMM4, %YMM0, %k1

> > +     kmovd   %k1, %ecx

> > +     /* Restore s1 pointer to rdi.  */

> > +     movq    %rdx, %rdi

> > +     testl   %ecx, %ecx

> > +     jnz     L(8x_return_vec_0_1_2_3)

> > +     /* NB: eax must be zero to reach here.  */

> > +     ret

> > +

> > +     /* Only entry is from L(more_8x_vec).  */

> > +     .p2align 4

> > +L(8x_last_2x_vec):

> > +     VPCMP   $4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(8x_return_vec_2)

> > +     /* Naturally aligned to 16 bytes.  */

> > +L(8x_last_1x_vec):

> > +     VMOVU   (VEC_SIZE * 3)(%rsi, %rdx), %YMM1

> > +     VPCMP   $4, (VEC_SIZE * 3)(%rdx), %YMM1, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(8x_return_vec_3)

> > +     ret

> > +

> > +     .p2align 4

> > +L(last_2x_vec):

> > +     /* Check second to last VEC.  */

> > +     VMOVU   -(VEC_SIZE * 2)(%rsi, %rdx, CHAR_SIZE), %YMM1

> > +     VPCMP   $4, -(VEC_SIZE * 2)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_1_end)

> > +

> > +     /* Check last VEC.  */

> > +     .p2align 4

> > +L(last_1x_vec):

> > +     VMOVU   -(VEC_SIZE * 1)(%rsi, %rdx, CHAR_SIZE), %YMM1

> > +     VPCMP   $4, -(VEC_SIZE * 1)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_0_end)

> >       ret

> > +

> > +     .p2align 4

> > +L(8x_return_vec_2):

> > +     subq    $VEC_SIZE, %rdx

> > +L(8x_return_vec_3):

> > +     tzcntl  %eax, %eax

> > +# ifdef USE_AS_WMEMCMP

> > +     leaq    (%rdx, %rax, CHAR_SIZE), %rax

> > +     movl    (VEC_SIZE * 3)(%rax), %ecx

> > +     xorl    %edx, %edx

> > +     cmpl    (VEC_SIZE * 3)(%rsi, %rax), %ecx

> > +     setg    %dl

> > +     leal    -1(%rdx, %rdx), %eax

> >  # else

> > +     addq    %rdx, %rax

> > +     movzbl  (VEC_SIZE * 3)(%rsi, %rax), %ecx

> > +     movzbl  (VEC_SIZE * 3)(%rax), %eax

> > +     subl    %ecx, %eax

> > +# endif

> > +     ret

> > +

> >       .p2align 4

> > -L(between_4_7):

> > -     /* Load as big endian with overlapping movbe to avoid branches.  */

> > -     movbe   (%rdi), %eax

> > -     movbe   (%rsi), %ecx

> > -     shlq    $32, %rax

> > -     shlq    $32, %rcx

> > -     movbe   -4(%rdi, %rdx), %edi

> > -     movbe   -4(%rsi, %rdx), %esi

> > -     orq     %rdi, %rax

> > -     orq     %rsi, %rcx

> > -     subq    %rcx, %rax

> > -     je      L(exit)

> > -     sbbl    %eax, %eax

> > -     orl     $1, %eax

> > +L(return_vec_0_end):

> > +     tzcntl  %eax, %eax

> > +     addl    %edx, %eax

> > +# ifdef USE_AS_WMEMCMP

> > +     movl    -VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx

> > +     xorl    %edx, %edx

> > +     cmpl    -VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx

> > +     setg    %dl

> > +     leal    -1(%rdx, %rdx), %eax

> > +# else

> > +     movzbl  -VEC_SIZE(%rsi, %rax), %ecx

> > +     movzbl  -VEC_SIZE(%rdi, %rax), %eax

> > +     subl    %ecx, %eax

> > +# endif

> >       ret

> >

> >       .p2align 4

> > -L(exit):

> > +L(return_vec_1_end):

> > +     tzcntl  %eax, %eax

> > +     addl    %edx, %eax

> > +# ifdef USE_AS_WMEMCMP

> > +     movl    -(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx

> > +     xorl    %edx, %edx

> > +     cmpl    -(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx

> > +     setg    %dl

> > +     leal    -1(%rdx, %rdx), %eax

> > +# else

> > +     movzbl  -(VEC_SIZE * 2)(%rsi, %rax), %ecx

> > +     movzbl  -(VEC_SIZE * 2)(%rdi, %rax), %eax

> > +     subl    %ecx, %eax

> > +# endif

> >       ret

> >

> > +

> >       .p2align 4

> > +L(page_cross_less_vec):

> > +     /* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28

> > +        bytes.  */

> > +     cmpl    $(16 / CHAR_SIZE), %edx

> > +     jae     L(between_16_31)

> > +# ifndef USE_AS_WMEMCMP

> > +     cmpl    $8, %edx

> > +     jae     L(between_8_15)

> > +     cmpl    $4, %edx

> > +     jae     L(between_4_7)

> >  L(between_2_3):

> >       /* Load as big endian to avoid branches.  */

> >       movzwl  (%rdi), %eax

> > @@ -217,224 +448,99 @@ L(between_2_3):

> >       shll    $8, %ecx

> >       bswap   %eax

> >       bswap   %ecx

> > -     movb    -1(%rdi, %rdx), %al

> > -     movb    -1(%rsi, %rdx), %cl

> > +     movzbl  -1(%rdi, %rdx), %edi

> > +     movzbl  -1(%rsi, %rdx), %esi

> > +     orl     %edi, %eax

> > +     orl     %esi, %ecx

> >       /* Subtraction is okay because the upper 8 bits are zero.  */

> >       subl    %ecx, %eax

> >       ret

> > -

> >       .p2align 4

> > -L(1):

> > -     movzbl  (%rdi), %eax

> > +L(one_or_less):

> > +     jb      L(zero)

> >       movzbl  (%rsi), %ecx

> > +     movzbl  (%rdi), %eax

> >       subl    %ecx, %eax

> >       ret

> > -# endif

> > -

> > -     .p2align 4

> > -L(zero):

> > -     xorl    %eax, %eax

> > -     ret

> >

> >       .p2align 4

> > -L(less_vec):

> > -# ifdef USE_AS_WMEMCMP

> > -     /* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes.  */

> > -     cmpb    $4, %dl

> > -     je      L(4)

> > -     jb      L(zero)

> > -# else

> > -     cmpb    $1, %dl

> > -     je      L(1)

> > -     jb      L(zero)

> > -     cmpb    $4, %dl

> > -     jb      L(between_2_3)

> > -     cmpb    $8, %dl

> > -     jb      L(between_4_7)

> > +L(between_8_15):

> >  # endif

> > -     cmpb    $16, %dl

> > -     jae     L(between_16_31)

> > -     /* It is between 8 and 15 bytes.  */

> > +     /* If USE_AS_WMEMCMP fall through into 8-15 byte case.  */

> >       vmovq   (%rdi), %XMM1

> >       vmovq   (%rsi), %XMM2

> > -     VPCMPEQ %XMM1, %XMM2, %k2

> > -     kmovw   %k2, %eax

> > -     subl    $XMM_MASK, %eax

> > -     jnz     L(first_vec)

> > +     VPCMP   $4, %XMM1, %XMM2, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_0)

> >       /* Use overlapping loads to avoid branches.  */

> > -     leaq    -8(%rdi, %rdx), %rdi

> > -     leaq    -8(%rsi, %rdx), %rsi

> > +     leaq    -8(%rdi, %rdx, CHAR_SIZE), %rdi

> > +     leaq    -8(%rsi, %rdx, CHAR_SIZE), %rsi

> >       vmovq   (%rdi), %XMM1

> >       vmovq   (%rsi), %XMM2

> > -     VPCMPEQ %XMM1, %XMM2, %k2

> > -     kmovw   %k2, %eax

> > -     subl    $XMM_MASK, %eax

> > -     jnz     L(first_vec)

> > +     VPCMP   $4, %XMM1, %XMM2, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_0)

> >       ret

> >

> >       .p2align 4

> > -L(between_16_31):

> > -     /* From 16 to 31 bytes.  No branch when size == 16.  */

> > -     VMOVU   (%rsi), %XMM2

> > -     VPCMPEQ (%rdi), %XMM2, %k2

> > -     kmovw   %k2, %eax

> > -     subl    $XMM_MASK, %eax

> > -     jnz     L(first_vec)

> > -

> > -     /* Use overlapping loads to avoid branches.  */

> > -     leaq    -16(%rdi, %rdx), %rdi

> > -     leaq    -16(%rsi, %rdx), %rsi

> > -     VMOVU   (%rsi), %XMM2

> > -     VPCMPEQ (%rdi), %XMM2, %k2

> > -     kmovw   %k2, %eax

> > -     subl    $XMM_MASK, %eax

> > -     jnz     L(first_vec)

> > +L(zero):

> > +     xorl    %eax, %eax

> >       ret

> >

> >       .p2align 4

> > -L(more_8x_vec):

> > -     /* More than 8 * VEC.  Check the first VEC.  */

> > -     VMOVU   (%rsi), %YMM2

> > -     VPCMPEQ (%rdi), %YMM2, %k2

> > -     kmovd   %k2, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> > -

> > -     /* Align the first memory area for aligned loads in the loop.

> > -        Compute how much the first memory area is misaligned.  */

> > -     movq    %rdi, %rcx

> > -     andl    $(VEC_SIZE - 1), %ecx

> > -     /* Get the negative of offset for alignment.  */

> > -     subq    $VEC_SIZE, %rcx

> > -     /* Adjust the second memory area.  */

> > -     subq    %rcx, %rsi

> > -     /* Adjust the first memory area which should be aligned now.  */

> > -     subq    %rcx, %rdi

> > -     /* Adjust length.  */

> > -     addq    %rcx, %rdx

> > -

> > -L(loop_4x_vec):

> > -     /* Compare 4 * VEC at a time forward.  */

> > -     VMOVU   (%rsi), %YMM1

> > -     VPCMPEQ (%rdi), %YMM1, %k1

> > -

> > -     VMOVU   VEC_SIZE(%rsi), %YMM2

> > -     VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2

> > -     kandd   %k2, %k1, %k5

> > -

> > -     VMOVU   (VEC_SIZE * 2)(%rsi), %YMM3

> > -     VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3

> > -     kandd   %k3, %k5, %k5

> > -

> > -     VMOVU   (VEC_SIZE * 3)(%rsi), %YMM4

> > -     VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4

> > -     kandd   %k4, %k5, %k5

> > -

> > -     kmovd   %k5, %eax

> > -     cmpl    $VEC_MASK, %eax

> > -     jne     L(4x_vec_end)

> > -

> > -     addq    $(VEC_SIZE * 4), %rdi

> > -     addq    $(VEC_SIZE * 4), %rsi

> > -

> > -     subq    $(VEC_SIZE * 4), %rdx

> > -     cmpq    $(VEC_SIZE * 4), %rdx

> > -     jae     L(loop_4x_vec)

> > -

> > -     /* Less than 4 * VEC.  */

> > -     cmpq    $VEC_SIZE, %rdx

> > -     jbe     L(last_vec)

> > -     cmpq    $(VEC_SIZE * 2), %rdx

> > -     jbe     L(last_2x_vec)

> > -

> > -L(last_4x_vec):

> > -     /* From 2 * VEC to 4 * VEC. */

> > -     VMOVU   (%rsi), %YMM2

> > -     VPCMPEQ (%rdi), %YMM2, %k2

> > -     kmovd   %k2, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> > -

> > -     addq    $VEC_SIZE, %rdi

> > -     addq    $VEC_SIZE, %rsi

> > -     VMOVU   (%rsi), %YMM2

> > -     VPCMPEQ (%rdi), %YMM2, %k2

> > -     kmovd   %k2, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> > +L(between_16_31):

> > +     /* From 16 to 31 bytes.  No branch when size == 16.  */

> > +     VMOVU   (%rsi), %XMM2

> > +     VPCMP   $4, (%rdi), %XMM2, %k1

> > +     kmovd   %k1, %eax

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_0)

> >

> >       /* Use overlapping loads to avoid branches.  */

> > -     leaq    -(3 * VEC_SIZE)(%rdi, %rdx), %rdi

> > -     leaq    -(3 * VEC_SIZE)(%rsi, %rdx), %rsi

> > -     VMOVU   (%rsi), %YMM2

> > -     VPCMPEQ (%rdi), %YMM2, %k2

> > -     kmovd   %k2, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> >

> > -     addq    $VEC_SIZE, %rdi

> > -     addq    $VEC_SIZE, %rsi

> > -     VMOVU   (%rsi), %YMM2

> > -     VPCMPEQ (%rdi), %YMM2, %k2

> > -     kmovd   %k2, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> > -     ret

> > -

> > -     .p2align 4

> > -L(4x_vec_end):

> > +     VMOVU   -16(%rsi, %rdx, CHAR_SIZE), %XMM2

> > +     leaq    -16(%rdi, %rdx, CHAR_SIZE), %rdi

> > +     leaq    -16(%rsi, %rdx, CHAR_SIZE), %rsi

> > +     VPCMP   $4, (%rdi), %XMM2, %k1

> >       kmovd   %k1, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec)

> > -     kmovd   %k2, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec_x1)

> > -     kmovd   %k3, %eax

> > -     subl    $VEC_MASK, %eax

> > -     jnz     L(first_vec_x2)

> > -     kmovd   %k4, %eax

> > -     subl    $VEC_MASK, %eax

> > -     tzcntl  %eax, %ecx

> > -# ifdef USE_AS_WMEMCMP

> > -     xorl    %eax, %eax

> > -     movl    (VEC_SIZE * 3)(%rdi, %rcx, 4), %edx

> > -     cmpl    (VEC_SIZE * 3)(%rsi, %rcx, 4), %edx

> > -     jmp     L(wmemcmp_return)

> > -# else

> > -     movzbl  (VEC_SIZE * 3)(%rdi, %rcx), %eax

> > -     movzbl  (VEC_SIZE * 3)(%rsi, %rcx), %edx

> > -     sub     %edx, %eax

> > -# endif

> > +     testl   %eax, %eax

> > +     jnz     L(return_vec_0)

> >       ret

> >

> > -     .p2align 4

> > -L(first_vec_x1):

> > -     tzcntl  %eax, %ecx

> >  # ifdef USE_AS_WMEMCMP

> > -     xorl    %eax, %eax

> > -     movl    VEC_SIZE(%rdi, %rcx, 4), %edx

> > -     cmpl    VEC_SIZE(%rsi, %rcx, 4), %edx

> > -     jmp     L(wmemcmp_return)

> > -# else

> > -     movzbl  VEC_SIZE(%rdi, %rcx), %eax

> > -     movzbl  VEC_SIZE(%rsi, %rcx), %edx

> > -     sub     %edx, %eax

> > -# endif

> > +     .p2align 4

> > +L(one_or_less):

> > +     jb      L(zero)

> > +     movl    (%rdi), %ecx

> > +     xorl    %edx, %edx

> > +     cmpl    (%rsi), %ecx

> > +     je      L(zero)

> > +     setg    %dl

> > +     leal    -1(%rdx, %rdx), %eax

> >       ret

> > +# else

> >

> >       .p2align 4

> > -L(first_vec_x2):

> > -     tzcntl  %eax, %ecx

> > -# ifdef USE_AS_WMEMCMP

> > -     xorl    %eax, %eax

> > -     movl    (VEC_SIZE * 2)(%rdi, %rcx, 4), %edx

> > -     cmpl    (VEC_SIZE * 2)(%rsi, %rcx, 4), %edx

> > -     jmp     L(wmemcmp_return)

> > -# else

> > -     movzbl  (VEC_SIZE * 2)(%rdi, %rcx), %eax

> > -     movzbl  (VEC_SIZE * 2)(%rsi, %rcx), %edx

> > -     sub     %edx, %eax

> > -# endif

> > +L(between_4_7):

> > +     /* Load as big endian with overlapping movbe to avoid branches.

> > +      */

> > +     movbe   (%rdi), %eax

> > +     movbe   (%rsi), %ecx

> > +     shlq    $32, %rax

> > +     shlq    $32, %rcx

> > +     movbe   -4(%rdi, %rdx), %edi

> > +     movbe   -4(%rsi, %rdx), %esi

> > +     orq     %rdi, %rax

> > +     orq     %rsi, %rcx

> > +     subq    %rcx, %rax

> > +     jz      L(zero_4_7)

> > +     sbbl    %eax, %eax

> > +     orl     $1, %eax

> > +L(zero_4_7):

> >       ret

> > +# endif

> > +

> >  END (MEMCMP)

> >  #endif

> > --

> > 2.29.2

> >

>

> OK, pending ifunc-impl-list.c change to require BMI2 for AVX2/EVEX

> memcmp.


Fixed in the AVX2 patch.

>

> Thanks.

>

> H.J.

Patch

diff --git a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
index 9c093972e1..654dc7ac8c 100644
--- a/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
+++ b/sysdeps/x86_64/multiarch/memcmp-evex-movbe.S
@@ -19,17 +19,22 @@ 
 #if IS_IN (libc)
 
 /* memcmp/wmemcmp is implemented as:
-   1. For size from 2 to 7 bytes, load as big endian with movbe and bswap
-      to avoid branches.
-   2. Use overlapping compare to avoid branch.
-   3. Use vector compare when size >= 4 bytes for memcmp or size >= 8
-      bytes for wmemcmp.
-   4. If size is 8 * VEC_SIZE or less, unroll the loop.
-   5. Compare 4 * VEC_SIZE at a time with the aligned first memory
+   1. Use ymm vector compares when possible. The only case where
+      vector compares is not possible for when size < CHAR_PER_VEC
+      and loading from either s1 or s2 would cause a page cross.
+   2. For size from 2 to 7 bytes on page cross, load as big endian
+      with movbe and bswap to avoid branches.
+   3. Use xmm vector compare when size >= 4 bytes for memcmp or
+      size >= 8 bytes for wmemcmp.
+   4. Optimistically compare up to first 4 * CHAR_PER_VEC one at a
+      to check for early mismatches. Only do this if its guranteed the
+      work is not wasted.
+   5. If size is 8 * VEC_SIZE or less, unroll the loop.
+   6. Compare 4 * VEC_SIZE at a time with the aligned first memory
       area.
-   6. Use 2 vector compares when size is 2 * VEC_SIZE or less.
-   7. Use 4 vector compares when size is 4 * VEC_SIZE or less.
-   8. Use 8 vector compares when size is 8 * VEC_SIZE or less.  */
+   7. Use 2 vector compares when size is 2 * CHAR_PER_VEC or less.
+   8. Use 4 vector compares when size is 4 * CHAR_PER_VEC or less.
+   9. Use 8 vector compares when size is 8 * CHAR_PER_VEC or less.  */
 
 # include <sysdep.h>
 
@@ -40,11 +45,21 @@ 
 # define VMOVU		vmovdqu64
 
 # ifdef USE_AS_WMEMCMP
-#  define VPCMPEQ	vpcmpeqd
+#  define CHAR_SIZE	4
+#  define VPCMP	vpcmpd
 # else
-#  define VPCMPEQ	vpcmpeqb
+#  define CHAR_SIZE	1
+#  define VPCMP	vpcmpub
 # endif
 
+# define VEC_SIZE	32
+# define PAGE_SIZE	4096
+# define CHAR_PER_VEC	(VEC_SIZE / CHAR_SIZE)
+
+# define XMM0		xmm16
+# define XMM1		xmm17
+# define XMM2		xmm18
+# define YMM0		ymm16
 # define XMM1		xmm17
 # define XMM2		xmm18
 # define YMM1		ymm17
@@ -54,15 +69,6 @@ 
 # define YMM5		ymm21
 # define YMM6		ymm22
 
-# define VEC_SIZE 32
-# ifdef USE_AS_WMEMCMP
-#  define VEC_MASK 0xff
-#  define XMM_MASK 0xf
-# else
-#  define VEC_MASK 0xffffffff
-#  define XMM_MASK 0xffff
-# endif
-
 /* Warning!
            wmemcmp has to use SIGNED comparison for elements.
            memcmp has to use UNSIGNED comparison for elemnts.
@@ -70,145 +76,370 @@ 
 
 	.section .text.evex,"ax",@progbits
 ENTRY (MEMCMP)
-# ifdef USE_AS_WMEMCMP
-	shl	$2, %RDX_LP
-# elif defined __ILP32__
+# ifdef __ILP32__
 	/* Clear the upper 32 bits.  */
 	movl	%edx, %edx
 # endif
-	cmp	$VEC_SIZE, %RDX_LP
+	cmp	$CHAR_PER_VEC, %RDX_LP
 	jb	L(less_vec)
 
 	/* From VEC to 2 * VEC.  No branch when size == VEC_SIZE.  */
-	VMOVU	(%rsi), %YMM2
-	VPCMPEQ (%rdi), %YMM2, %k1
+	VMOVU	(%rsi), %YMM1
+	/* Use compare not equals to directly check for mismatch.  */
+	VPCMP	$4, (%rdi), %YMM1, %k1
 	kmovd	%k1, %eax
-	subl    $VEC_MASK, %eax
-	jnz	L(first_vec)
-
-	cmpq	$(VEC_SIZE * 2), %rdx
-	jbe	L(last_vec)
-
-	/* More than 2 * VEC.  */
-	cmpq	$(VEC_SIZE * 8), %rdx
-	ja	L(more_8x_vec)
-	cmpq	$(VEC_SIZE * 4), %rdx
-	jb	L(last_4x_vec)
+	/* NB: eax must be destination register if going to
+	   L(return_vec_[0,2]). For L(return_vec_3 destination register
+	   must be ecx.  */
+	testl	%eax, %eax
+	jnz	L(return_vec_0)
 
-	/* From 4 * VEC to 8 * VEC, inclusively. */
-	VMOVU	(%rsi), %YMM1
-	VPCMPEQ (%rdi), %YMM1, %k1
+	cmpq	$(CHAR_PER_VEC * 2), %rdx
+	jbe	L(last_1x_vec)
 
+	/* Check second VEC no matter what.  */
 	VMOVU	VEC_SIZE(%rsi), %YMM2
-	VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
+	VPCMP	$4, VEC_SIZE(%rdi), %YMM2, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_1)
+
+	/* Less than 4 * VEC.  */
+	cmpq	$(CHAR_PER_VEC * 4), %rdx
+	jbe	L(last_2x_vec)
 
+	/* Check third and fourth VEC no matter what.  */
 	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM3
-	VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
+	VPCMP	$4, (VEC_SIZE * 2)(%rdi), %YMM3, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_2)
 
 	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM4
-	VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
+	VPCMP	$4, (VEC_SIZE * 3)(%rdi), %YMM4, %k1
+	kmovd	%k1, %ecx
+	testl	%ecx, %ecx
+	jnz	L(return_vec_3)
 
-	kandd	%k1, %k2, %k5
-	kandd	%k3, %k4, %k6
-	kandd	%k5, %k6, %k6
+	/* Zero YMM0. 4x VEC reduction is done with vpxor + vtern so
+	   compare with zero to get a mask is needed.  */
+	vpxorq	%XMM0, %XMM0, %XMM0
 
-	kmovd	%k6, %eax
-	cmpl	$VEC_MASK, %eax
-	jne	L(4x_vec_end)
+	/* Go to 4x VEC loop.  */
+	cmpq	$(CHAR_PER_VEC * 8), %rdx
+	ja	L(more_8x_vec)
 
-	leaq	-(4 * VEC_SIZE)(%rdi, %rdx), %rdi
-	leaq	-(4 * VEC_SIZE)(%rsi, %rdx), %rsi
-	VMOVU	(%rsi), %YMM1
-	VPCMPEQ (%rdi), %YMM1, %k1
+	/* Handle remainder of size = 4 * VEC + 1 to 8 * VEC without any
+	   branches.  */
 
-	VMOVU	VEC_SIZE(%rsi), %YMM2
-	VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
-	kandd	%k1, %k2, %k5
+	/* Load first two VEC from s2 before adjusting addresses.  */
+	VMOVU	-(VEC_SIZE * 4)(%rsi, %rdx, CHAR_SIZE), %YMM1
+	VMOVU	-(VEC_SIZE * 3)(%rsi, %rdx, CHAR_SIZE), %YMM2
+	leaq	-(4 * VEC_SIZE)(%rdi, %rdx, CHAR_SIZE), %rdi
+	leaq	-(4 * VEC_SIZE)(%rsi, %rdx, CHAR_SIZE), %rsi
+
+	/* Wait to load from s1 until addressed adjust due to
+	   unlamination of microfusion with complex address mode.  */
+
+	/* vpxor will be all 0s if s1 and s2 are equal. Otherwise it
+	   will have some 1s.  */
+	vpxorq	(%rdi), %YMM1, %YMM1
+	vpxorq	(VEC_SIZE)(%rdi), %YMM2, %YMM2
 
 	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM3
-	VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
-	kandd	%k3, %k5, %k5
+	vpxorq	(VEC_SIZE * 2)(%rdi), %YMM3, %YMM3
+	/* Or together YMM1, YMM2, and YMM3 into YMM3.  */
+	vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
 
 	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM4
-	VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
-	kandd	%k4, %k5, %k5
+	/* Ternary logic to xor (VEC_SIZE * 3)(%rdi) with YMM4 while
+	   oring with YMM3. Result is stored in YMM4.  */
+	vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4
+	/* Compare YMM4 with 0. If any 1s s1 and s2 don't match.  */
+	VPCMP	$4, %YMM4, %YMM0, %k1
+	kmovd	%k1, %ecx
+	testl	%ecx, %ecx
+	jnz	L(return_vec_0_1_2_3)
+	/* NB: eax must be zero to reach here.  */
+	ret
 
-	kmovd	%k5, %eax
-	cmpl	$VEC_MASK, %eax
-	jne	L(4x_vec_end)
-	xorl	%eax, %eax
+	/* NB: aligning 32 here allows for the rest of the jump targets
+	   to be tuned for 32 byte alignment. Most important this ensures
+	   the L(more_8x_vec) loop is 32 byte aligned.  */
+	.p2align 5
+L(less_vec):
+	/* Check if one or less CHAR. This is necessary for size = 0 but
+	   is also faster for size = CHAR_SIZE.  */
+	cmpl	$1, %edx
+	jbe	L(one_or_less)
+
+	/* Check if loading one VEC from either s1 or s2 could cause a
+	   page cross. This can have false positives but is by far the
+	   fastest method.  */
+	movl	%edi, %eax
+	orl	%esi, %eax
+	andl	$(PAGE_SIZE - 1), %eax
+	cmpl	$(PAGE_SIZE - VEC_SIZE), %eax
+	jg	L(page_cross_less_vec)
+
+	/* No page cross possible.  */
+	VMOVU	(%rsi), %YMM2
+	VPCMP	$4, (%rdi), %YMM2, %k1
+	kmovd	%k1, %eax
+	/* Create mask in ecx for potentially in bound matches.  */
+	bzhil	%edx, %eax, %eax
+	jnz	L(return_vec_0)
 	ret
 
 	.p2align 4
-L(last_2x_vec):
-	/* From VEC to 2 * VEC.  No branch when size == VEC_SIZE.  */
-	VMOVU	(%rsi), %YMM2
-	VPCMPEQ (%rdi), %YMM2, %k2
-	kmovd	%k2, %eax
-	subl    $VEC_MASK, %eax
-	jnz	L(first_vec)
+L(return_vec_0):
+	tzcntl	%eax, %eax
+# ifdef USE_AS_WMEMCMP
+	movl	(%rdi, %rax, CHAR_SIZE), %ecx
+	xorl	%edx, %edx
+	cmpl	(%rsi, %rax, CHAR_SIZE), %ecx
+	/* NB: no partial register stall here because xorl zero idiom
+	   above.  */
+	setg	%dl
+	leal	-1(%rdx, %rdx), %eax
+# else
+	movzbl	(%rsi, %rax), %ecx
+	movzbl	(%rdi, %rax), %eax
+	subl	%ecx, %eax
+# endif
+	ret
 
-L(last_vec):
-	/* Use overlapping loads to avoid branches.  */
-	leaq	-VEC_SIZE(%rdi, %rdx), %rdi
-	leaq	-VEC_SIZE(%rsi, %rdx), %rsi
-	VMOVU	(%rsi), %YMM2
-	VPCMPEQ (%rdi), %YMM2, %k2
-	kmovd	%k2, %eax
-	subl    $VEC_MASK, %eax
-	jnz	L(first_vec)
+	/* NB: No p2align necessary. Alignment  % 16 is naturally 1
+	   which is good enough for a target not in a loop.  */
+L(return_vec_1):
+	tzcntl	%eax, %eax
+# ifdef USE_AS_WMEMCMP
+	movl	VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx
+	xorl	%edx, %edx
+	cmpl	VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx
+	setg	%dl
+	leal	-1(%rdx, %rdx), %eax
+# else
+	movzbl	VEC_SIZE(%rsi, %rax), %ecx
+	movzbl	VEC_SIZE(%rdi, %rax), %eax
+	subl	%ecx, %eax
+# endif
 	ret
 
-	.p2align 4
-L(first_vec):
-	/* A byte or int32 is different within 16 or 32 bytes.  */
-	tzcntl	%eax, %ecx
+	/* NB: No p2align necessary. Alignment  % 16 is naturally 2
+	   which is good enough for a target not in a loop.  */
+L(return_vec_2):
+	tzcntl	%eax, %eax
 # ifdef USE_AS_WMEMCMP
-	xorl	%eax, %eax
-	movl	(%rdi, %rcx, 4), %edx
-	cmpl	(%rsi, %rcx, 4), %edx
-L(wmemcmp_return):
-	setl	%al
-	negl	%eax
-	orl	$1, %eax
+	movl	(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx
+	xorl	%edx, %edx
+	cmpl	(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx
+	setg	%dl
+	leal	-1(%rdx, %rdx), %eax
 # else
-	movzbl	(%rdi, %rcx), %eax
-	movzbl	(%rsi, %rcx), %edx
-	sub	%edx, %eax
+	movzbl	(VEC_SIZE * 2)(%rsi, %rax), %ecx
+	movzbl	(VEC_SIZE * 2)(%rdi, %rax), %eax
+	subl	%ecx, %eax
 # endif
 	ret
 
+	.p2align 4
+L(8x_return_vec_0_1_2_3):
+	/* Returning from L(more_8x_vec) requires restoring rsi.  */
+	addq	%rdi, %rsi
+L(return_vec_0_1_2_3):
+	VPCMP	$4, %YMM1, %YMM0, %k0
+	kmovd	%k0, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_0)
+
+	VPCMP	$4, %YMM2, %YMM0, %k0
+	kmovd	%k0, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_1)
+
+	VPCMP	$4, %YMM3, %YMM0, %k0
+	kmovd	%k0, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_2)
+L(return_vec_3):
+	tzcntl	%ecx, %ecx
 # ifdef USE_AS_WMEMCMP
+	movl	(VEC_SIZE * 3)(%rdi, %rcx, CHAR_SIZE), %eax
+	xorl	%edx, %edx
+	cmpl	(VEC_SIZE * 3)(%rsi, %rcx, CHAR_SIZE), %eax
+	setg	%dl
+	leal	-1(%rdx, %rdx), %eax
+# else
+	movzbl	(VEC_SIZE * 3)(%rdi, %rcx), %eax
+	movzbl	(VEC_SIZE * 3)(%rsi, %rcx), %ecx
+	subl	%ecx, %eax
+# endif
+	ret
+
 	.p2align 4
-L(4):
-	xorl	%eax, %eax
-	movl	(%rdi), %edx
-	cmpl	(%rsi), %edx
-	jne	L(wmemcmp_return)
+L(more_8x_vec):
+	/* Set end of s1 in rdx.  */
+	leaq	-(VEC_SIZE * 4)(%rdi, %rdx, CHAR_SIZE), %rdx
+	/* rsi stores s2 - s1. This allows loop to only update one
+	   pointer.  */
+	subq	%rdi, %rsi
+	/* Align s1 pointer.  */
+	andq	$-VEC_SIZE, %rdi
+	/* Adjust because first 4x vec where check already.  */
+	subq	$-(VEC_SIZE * 4), %rdi
+	.p2align 4
+L(loop_4x_vec):
+	VMOVU	(%rsi, %rdi), %YMM1
+	vpxorq	(%rdi), %YMM1, %YMM1
+
+	VMOVU	VEC_SIZE(%rsi, %rdi), %YMM2
+	vpxorq	VEC_SIZE(%rdi), %YMM2, %YMM2
+
+	VMOVU	(VEC_SIZE * 2)(%rsi, %rdi), %YMM3
+	vpxorq	(VEC_SIZE * 2)(%rdi), %YMM3, %YMM3
+	vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
+
+	VMOVU	(VEC_SIZE * 3)(%rsi, %rdi), %YMM4
+	vpternlogd $0xde, (VEC_SIZE * 3)(%rdi), %YMM3, %YMM4
+	VPCMP	$4, %YMM4, %YMM0, %k1
+	kmovd	%k1, %ecx
+	testl	%ecx, %ecx
+	jnz	L(8x_return_vec_0_1_2_3)
+	subq	$-(VEC_SIZE * 4), %rdi
+	cmpq	%rdx, %rdi
+	jb	L(loop_4x_vec)
+
+	subq	%rdx, %rdi
+	/* rdi has 4 * VEC_SIZE - remaining length.  */
+	cmpl	$(VEC_SIZE * 3), %edi
+	jae	L(8x_last_1x_vec)
+	/* Load regardless of branch.  */
+	VMOVU	(VEC_SIZE * 2)(%rsi, %rdx), %YMM3
+	cmpl	$(VEC_SIZE * 2), %edi
+	jae	L(8x_last_2x_vec)
+
+	VMOVU	(%rsi, %rdx), %YMM1
+	vpxorq	(%rdx), %YMM1, %YMM1
+
+	VMOVU	VEC_SIZE(%rsi, %rdx), %YMM2
+	vpxorq	VEC_SIZE(%rdx), %YMM2, %YMM2
+
+	vpxorq	(VEC_SIZE * 2)(%rdx), %YMM3, %YMM3
+	vpternlogd $0xfe, %YMM1, %YMM2, %YMM3
+
+	VMOVU	(VEC_SIZE * 3)(%rsi, %rdx), %YMM4
+	vpternlogd $0xde, (VEC_SIZE * 3)(%rdx), %YMM3, %YMM4
+	VPCMP	$4, %YMM4, %YMM0, %k1
+	kmovd	%k1, %ecx
+	/* Restore s1 pointer to rdi.  */
+	movq	%rdx, %rdi
+	testl	%ecx, %ecx
+	jnz	L(8x_return_vec_0_1_2_3)
+	/* NB: eax must be zero to reach here.  */
+	ret
+
+	/* Only entry is from L(more_8x_vec).  */
+	.p2align 4
+L(8x_last_2x_vec):
+	VPCMP	$4, (VEC_SIZE * 2)(%rdx), %YMM3, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(8x_return_vec_2)
+	/* Naturally aligned to 16 bytes.  */
+L(8x_last_1x_vec):
+	VMOVU	(VEC_SIZE * 3)(%rsi, %rdx), %YMM1
+	VPCMP	$4, (VEC_SIZE * 3)(%rdx), %YMM1, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(8x_return_vec_3)
+	ret
+
+	.p2align 4
+L(last_2x_vec):
+	/* Check second to last VEC.  */
+	VMOVU	-(VEC_SIZE * 2)(%rsi, %rdx, CHAR_SIZE), %YMM1
+	VPCMP	$4, -(VEC_SIZE * 2)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_1_end)
+
+	/* Check last VEC.  */
+	.p2align 4
+L(last_1x_vec):
+	VMOVU	-(VEC_SIZE * 1)(%rsi, %rdx, CHAR_SIZE), %YMM1
+	VPCMP	$4, -(VEC_SIZE * 1)(%rdi, %rdx, CHAR_SIZE), %YMM1, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_0_end)
 	ret
+
+	.p2align 4
+L(8x_return_vec_2):
+	subq	$VEC_SIZE, %rdx
+L(8x_return_vec_3):
+	tzcntl	%eax, %eax
+# ifdef USE_AS_WMEMCMP
+	leaq	(%rdx, %rax, CHAR_SIZE), %rax
+	movl	(VEC_SIZE * 3)(%rax), %ecx
+	xorl	%edx, %edx
+	cmpl	(VEC_SIZE * 3)(%rsi, %rax), %ecx
+	setg	%dl
+	leal	-1(%rdx, %rdx), %eax
 # else
+	addq	%rdx, %rax
+	movzbl	(VEC_SIZE * 3)(%rsi, %rax), %ecx
+	movzbl	(VEC_SIZE * 3)(%rax), %eax
+	subl	%ecx, %eax
+# endif
+	ret
+
 	.p2align 4
-L(between_4_7):
-	/* Load as big endian with overlapping movbe to avoid branches.  */
-	movbe	(%rdi), %eax
-	movbe	(%rsi), %ecx
-	shlq	$32, %rax
-	shlq	$32, %rcx
-	movbe	-4(%rdi, %rdx), %edi
-	movbe	-4(%rsi, %rdx), %esi
-	orq	%rdi, %rax
-	orq	%rsi, %rcx
-	subq	%rcx, %rax
-	je	L(exit)
-	sbbl	%eax, %eax
-	orl	$1, %eax
+L(return_vec_0_end):
+	tzcntl	%eax, %eax
+	addl	%edx, %eax
+# ifdef USE_AS_WMEMCMP
+	movl	-VEC_SIZE(%rdi, %rax, CHAR_SIZE), %ecx
+	xorl	%edx, %edx
+	cmpl	-VEC_SIZE(%rsi, %rax, CHAR_SIZE), %ecx
+	setg	%dl
+	leal	-1(%rdx, %rdx), %eax
+# else
+	movzbl	-VEC_SIZE(%rsi, %rax), %ecx
+	movzbl	-VEC_SIZE(%rdi, %rax), %eax
+	subl	%ecx, %eax
+# endif
 	ret
 
 	.p2align 4
-L(exit):
+L(return_vec_1_end):
+	tzcntl	%eax, %eax
+	addl	%edx, %eax
+# ifdef USE_AS_WMEMCMP
+	movl	-(VEC_SIZE * 2)(%rdi, %rax, CHAR_SIZE), %ecx
+	xorl	%edx, %edx
+	cmpl	-(VEC_SIZE * 2)(%rsi, %rax, CHAR_SIZE), %ecx
+	setg	%dl
+	leal	-1(%rdx, %rdx), %eax
+# else
+	movzbl	-(VEC_SIZE * 2)(%rsi, %rax), %ecx
+	movzbl	-(VEC_SIZE * 2)(%rdi, %rax), %eax
+	subl	%ecx, %eax
+# endif
 	ret
 
+
 	.p2align 4
+L(page_cross_less_vec):
+	/* if USE_AS_WMEMCMP it can only be 0, 4, 8, 12, 16, 20, 24, 28
+	   bytes.  */
+	cmpl	$(16 / CHAR_SIZE), %edx
+	jae	L(between_16_31)
+# ifndef USE_AS_WMEMCMP
+	cmpl	$8, %edx
+	jae	L(between_8_15)
+	cmpl	$4, %edx
+	jae	L(between_4_7)
 L(between_2_3):
 	/* Load as big endian to avoid branches.  */
 	movzwl	(%rdi), %eax
@@ -217,224 +448,99 @@  L(between_2_3):
 	shll	$8, %ecx
 	bswap	%eax
 	bswap	%ecx
-	movb	-1(%rdi, %rdx), %al
-	movb	-1(%rsi, %rdx), %cl
+	movzbl	-1(%rdi, %rdx), %edi
+	movzbl	-1(%rsi, %rdx), %esi
+	orl	%edi, %eax
+	orl	%esi, %ecx
 	/* Subtraction is okay because the upper 8 bits are zero.  */
 	subl	%ecx, %eax
 	ret
-
 	.p2align 4
-L(1):
-	movzbl	(%rdi), %eax
+L(one_or_less):
+	jb	L(zero)
 	movzbl	(%rsi), %ecx
+	movzbl	(%rdi), %eax
 	subl	%ecx, %eax
 	ret
-# endif
-
-	.p2align 4
-L(zero):
-	xorl	%eax, %eax
-	ret
 
 	.p2align 4
-L(less_vec):
-# ifdef USE_AS_WMEMCMP
-	/* It can only be 0, 4, 8, 12, 16, 20, 24, 28 bytes.  */
-	cmpb	$4, %dl
-	je	L(4)
-	jb	L(zero)
-# else
-	cmpb	$1, %dl
-	je	L(1)
-	jb	L(zero)
-	cmpb	$4, %dl
-	jb	L(between_2_3)
-	cmpb	$8, %dl
-	jb	L(between_4_7)
+L(between_8_15):
 # endif
-	cmpb	$16, %dl
-	jae	L(between_16_31)
-	/* It is between 8 and 15 bytes.  */
+	/* If USE_AS_WMEMCMP fall through into 8-15 byte case.  */
 	vmovq	(%rdi), %XMM1
 	vmovq	(%rsi), %XMM2
-	VPCMPEQ %XMM1, %XMM2, %k2
-	kmovw	%k2, %eax
-	subl    $XMM_MASK, %eax
-	jnz	L(first_vec)
+	VPCMP	$4, %XMM1, %XMM2, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_0)
 	/* Use overlapping loads to avoid branches.  */
-	leaq	-8(%rdi, %rdx), %rdi
-	leaq	-8(%rsi, %rdx), %rsi
+	leaq	-8(%rdi, %rdx, CHAR_SIZE), %rdi
+	leaq	-8(%rsi, %rdx, CHAR_SIZE), %rsi
 	vmovq	(%rdi), %XMM1
 	vmovq	(%rsi), %XMM2
-	VPCMPEQ %XMM1, %XMM2, %k2
-	kmovw	%k2, %eax
-	subl    $XMM_MASK, %eax
-	jnz	L(first_vec)
+	VPCMP	$4, %XMM1, %XMM2, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_0)
 	ret
 
 	.p2align 4
-L(between_16_31):
-	/* From 16 to 31 bytes.  No branch when size == 16.  */
-	VMOVU	(%rsi), %XMM2
-	VPCMPEQ (%rdi), %XMM2, %k2
-	kmovw	%k2, %eax
-	subl    $XMM_MASK, %eax
-	jnz	L(first_vec)
-
-	/* Use overlapping loads to avoid branches.  */
-	leaq	-16(%rdi, %rdx), %rdi
-	leaq	-16(%rsi, %rdx), %rsi
-	VMOVU	(%rsi), %XMM2
-	VPCMPEQ (%rdi), %XMM2, %k2
-	kmovw	%k2, %eax
-	subl    $XMM_MASK, %eax
-	jnz	L(first_vec)
+L(zero):
+	xorl	%eax, %eax
 	ret
 
 	.p2align 4
-L(more_8x_vec):
-	/* More than 8 * VEC.  Check the first VEC.  */
-	VMOVU	(%rsi), %YMM2
-	VPCMPEQ (%rdi), %YMM2, %k2
-	kmovd	%k2, %eax
-	subl    $VEC_MASK, %eax
-	jnz	L(first_vec)
-
-	/* Align the first memory area for aligned loads in the loop.
-	   Compute how much the first memory area is misaligned.  */
-	movq	%rdi, %rcx
-	andl	$(VEC_SIZE - 1), %ecx
-	/* Get the negative of offset for alignment.  */
-	subq	$VEC_SIZE, %rcx
-	/* Adjust the second memory area.  */
-	subq	%rcx, %rsi
-	/* Adjust the first memory area which should be aligned now.  */
-	subq	%rcx, %rdi
-	/* Adjust length.  */
-	addq	%rcx, %rdx
-
-L(loop_4x_vec):
-	/* Compare 4 * VEC at a time forward.  */
-	VMOVU	(%rsi), %YMM1
-	VPCMPEQ (%rdi), %YMM1, %k1
-
-	VMOVU	VEC_SIZE(%rsi), %YMM2
-	VPCMPEQ VEC_SIZE(%rdi), %YMM2, %k2
-	kandd	%k2, %k1, %k5
-
-	VMOVU	(VEC_SIZE * 2)(%rsi), %YMM3
-	VPCMPEQ (VEC_SIZE * 2)(%rdi), %YMM3, %k3
-	kandd	%k3, %k5, %k5
-
-	VMOVU	(VEC_SIZE * 3)(%rsi), %YMM4
-	VPCMPEQ (VEC_SIZE * 3)(%rdi), %YMM4, %k4
-	kandd	%k4, %k5, %k5
-
-	kmovd	%k5, %eax
-	cmpl	$VEC_MASK, %eax
-	jne	L(4x_vec_end)
-
-	addq	$(VEC_SIZE * 4), %rdi
-	addq	$(VEC_SIZE * 4), %rsi
-
-	subq	$(VEC_SIZE * 4), %rdx
-	cmpq	$(VEC_SIZE * 4), %rdx
-	jae	L(loop_4x_vec)
-
-	/* Less than 4 * VEC.  */
-	cmpq	$VEC_SIZE, %rdx
-	jbe	L(last_vec)
-	cmpq	$(VEC_SIZE * 2), %rdx
-	jbe	L(last_2x_vec)
-
-L(last_4x_vec):
-	/* From 2 * VEC to 4 * VEC. */
-	VMOVU	(%rsi), %YMM2
-	VPCMPEQ (%rdi), %YMM2, %k2
-	kmovd	%k2, %eax
-	subl    $VEC_MASK, %eax
-	jnz	L(first_vec)
-
-	addq	$VEC_SIZE, %rdi
-	addq	$VEC_SIZE, %rsi
-	VMOVU	(%rsi), %YMM2
-	VPCMPEQ (%rdi), %YMM2, %k2
-	kmovd	%k2, %eax
-	subl    $VEC_MASK, %eax
-	jnz	L(first_vec)
+L(between_16_31):
+	/* From 16 to 31 bytes.  No branch when size == 16.  */
+	VMOVU	(%rsi), %XMM2
+	VPCMP	$4, (%rdi), %XMM2, %k1
+	kmovd	%k1, %eax
+	testl	%eax, %eax
+	jnz	L(return_vec_0)
 
 	/* Use overlapping loads to avoid branches.  */
-	leaq	-(3 * VEC_SIZE)(%rdi, %rdx), %rdi
-	leaq	-(3 * VEC_SIZE)(%rsi, %rdx), %rsi
-	VMOVU	(%rsi), %YMM2
-	VPCMPEQ (%rdi), %YMM2, %k2
-	kmovd	%k2, %eax
-	subl    $VEC_MASK, %eax
-	jnz	L(first_vec)
 
-	addq	$VEC_SIZE, %rdi
-	addq	$VEC_SIZE, %rsi
-	VMOVU	(%rsi), %YMM2
-	VPCMPEQ (%rdi), %YMM2, %k2
-	kmovd	%k2, %eax
-	subl    $VEC_MASK, %eax
-	jnz	L(first_vec)
-	ret
-
-	.p2align 4
-L(4x_vec_end):
+	VMOVU	-16(%rsi, %rdx, CHAR_SIZE), %XMM2
+	leaq	-16(%rdi, %rdx, CHAR_SIZE), %rdi
+	leaq	-16(%rsi, %rdx, CHAR_SIZE), %rsi
+	VPCMP	$4, (%rdi), %XMM2, %k1
 	kmovd	%k1, %eax
-	subl	$VEC_MASK, %eax
-	jnz	L(first_vec)
-	kmovd	%k2, %eax
-	subl	$VEC_MASK, %eax
-	jnz	L(first_vec_x1)
-	kmovd	%k3, %eax
-	subl	$VEC_MASK, %eax
-	jnz	L(first_vec_x2)
-	kmovd	%k4, %eax
-	subl	$VEC_MASK, %eax
-	tzcntl	%eax, %ecx
-# ifdef USE_AS_WMEMCMP
-	xorl	%eax, %eax
-	movl	(VEC_SIZE * 3)(%rdi, %rcx, 4), %edx
-	cmpl	(VEC_SIZE * 3)(%rsi, %rcx, 4), %edx
-	jmp	L(wmemcmp_return)
-# else
-	movzbl	(VEC_SIZE * 3)(%rdi, %rcx), %eax
-	movzbl	(VEC_SIZE * 3)(%rsi, %rcx), %edx
-	sub	%edx, %eax
-# endif
+	testl	%eax, %eax
+	jnz	L(return_vec_0)
 	ret
 
-	.p2align 4
-L(first_vec_x1):
-	tzcntl	%eax, %ecx
 # ifdef USE_AS_WMEMCMP
-	xorl	%eax, %eax
-	movl	VEC_SIZE(%rdi, %rcx, 4), %edx
-	cmpl	VEC_SIZE(%rsi, %rcx, 4), %edx
-	jmp	L(wmemcmp_return)
-# else
-	movzbl	VEC_SIZE(%rdi, %rcx), %eax
-	movzbl	VEC_SIZE(%rsi, %rcx), %edx
-	sub	%edx, %eax
-# endif
+	.p2align 4
+L(one_or_less):
+	jb	L(zero)
+	movl	(%rdi), %ecx
+	xorl	%edx, %edx
+	cmpl	(%rsi), %ecx
+	je	L(zero)
+	setg	%dl
+	leal	-1(%rdx, %rdx), %eax
 	ret
+# else
 
 	.p2align 4
-L(first_vec_x2):
-	tzcntl	%eax, %ecx
-# ifdef USE_AS_WMEMCMP
-	xorl	%eax, %eax
-	movl	(VEC_SIZE * 2)(%rdi, %rcx, 4), %edx
-	cmpl	(VEC_SIZE * 2)(%rsi, %rcx, 4), %edx
-	jmp	L(wmemcmp_return)
-# else
-	movzbl	(VEC_SIZE * 2)(%rdi, %rcx), %eax
-	movzbl	(VEC_SIZE * 2)(%rsi, %rcx), %edx
-	sub	%edx, %eax
-# endif
+L(between_4_7):
+	/* Load as big endian with overlapping movbe to avoid branches.
+	 */
+	movbe	(%rdi), %eax
+	movbe	(%rsi), %ecx
+	shlq	$32, %rax
+	shlq	$32, %rcx
+	movbe	-4(%rdi, %rdx), %edi
+	movbe	-4(%rsi, %rdx), %esi
+	orq	%rdi, %rax
+	orq	%rsi, %rcx
+	subq	%rcx, %rax
+	jz	L(zero_4_7)
+	sbbl	%eax, %eax
+	orl	$1, %eax
+L(zero_4_7):
 	ret
+# endif
+
 END (MEMCMP)
 #endif