powerpc64le: Optimize memset for POWER10

Message ID 20210428144048.gyeulahanuzjiotq@work-tp
State Superseded
Headers show
Series
  • powerpc64le: Optimize memset for POWER10
Related show

Commit Message

Andreas Roeseler via Libc-alpha April 28, 2021, 2:40 p.m.
This implementation is based on __memset_power8 and integrates a lot
of suggestions from Anton Blanchard.

The biggest difference is that it makes extensive use of stxvl to
alignment and tail code to avoid branches and small stores.  It has
three main execution paths:

a) "Short lengths" for lengths up to 64 bytes, avoiding as many
   branches as possible.

b) "General case" for larger lengths, it has an alignment section
   using stxvl to avoid branches, a 128 bytes loop and then a tail
   code, again using stxvl with few branches.

c) "Zeroing cache blocks" for lengths from 256 bytes upwards and set
   value being zero.  It is mostly the __memset_power8 code but the
   alignment phase was simplified because, at this point, address is
   already 16-bytes aligned and also changed to use vector stores.
   The tail code was also simplified to reuse the general case tail.

All unaligned stores use stxvl instructions that do not generate
alignment interrupts on POWER10, making it safe to use on
caching-inhibited memory.

On average, this implementation provides something around 30%
improvement when compared to __memset_power8.
---
 sysdeps/powerpc/powerpc64/le/power10/memset.S | 251 ++++++++++++++++++
 sysdeps/powerpc/powerpc64/multiarch/Makefile  |   3 +-
 sysdeps/powerpc/powerpc64/multiarch/bzero.c   |   8 +
 .../powerpc64/multiarch/ifunc-impl-list.c     |  14 +
 .../powerpc64/multiarch/memset-power10.S      |  27 ++
 sysdeps/powerpc/powerpc64/multiarch/memset.c  |   8 +
 6 files changed, 310 insertions(+), 1 deletion(-)
 create mode 100644 sysdeps/powerpc/powerpc64/le/power10/memset.S
 create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

-- 
2.26.2

Comments

Andreas Roeseler via Libc-alpha April 28, 2021, 6:48 p.m. | #1
Thanks Raoni,
LGTM with some small comment fixes, all tests pass.

Quoting Raoni Fassina Firmino via Libc-alpha (2021-04-28 11:40:48)
> This implementation is based on __memset_power8 and integrates a lot

> of suggestions from Anton Blanchard.

> 

> The biggest difference is that it makes extensive use of stxvl to

> alignment and tail code to avoid branches and small stores.  It has

> three main execution paths:

> 

> a) "Short lengths" for lengths up to 64 bytes, avoiding as many

>    branches as possible.

> 

> b) "General case" for larger lengths, it has an alignment section

>    using stxvl to avoid branches, a 128 bytes loop and then a tail

>    code, again using stxvl with few branches.

> 

> c) "Zeroing cache blocks" for lengths from 256 bytes upwards and set

>    value being zero.  It is mostly the __memset_power8 code but the

>    alignment phase was simplified because, at this point, address is

>    already 16-bytes aligned and also changed to use vector stores.

>    The tail code was also simplified to reuse the general case tail.

> 

> All unaligned stores use stxvl instructions that do not generate

> alignment interrupts on POWER10, making it safe to use on

> caching-inhibited memory.

> 

> On average, this implementation provides something around 30%

> improvement when compared to __memset_power8.

> ---

>  sysdeps/powerpc/powerpc64/le/power10/memset.S | 251 ++++++++++++++++++

>  sysdeps/powerpc/powerpc64/multiarch/Makefile  |   3 +-

>  sysdeps/powerpc/powerpc64/multiarch/bzero.c   |   8 +

>  .../powerpc64/multiarch/ifunc-impl-list.c     |  14 +

>  .../powerpc64/multiarch/memset-power10.S      |  27 ++

>  sysdeps/powerpc/powerpc64/multiarch/memset.c  |   8 +

>  6 files changed, 310 insertions(+), 1 deletion(-)

>  create mode 100644 sysdeps/powerpc/powerpc64/le/power10/memset.S

>  create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> 

> diff --git a/sysdeps/powerpc/powerpc64/le/power10/memset.S b/sysdeps/powerpc/powerpc64/le/power10/memset.S

> new file mode 100644

> index 000000000000..c8b77fc64596

> --- /dev/null

> +++ b/sysdeps/powerpc/powerpc64/le/power10/memset.S

> @@ -0,0 +1,251 @@

> +/* Optimized memset implementation for PowerPC64/POWER10.

> +   Copyright (C) 2021 Free Software Foundation, Inc.

> +   This file is part of the GNU C Library.

> +

> +   The GNU C Library is free software; you can redistribute it and/or

> +   modify it under the terms of the GNU Lesser General Public

> +   License as published by the Free Software Foundation; either

> +   version 2.1 of the License, or (at your option) any later version.

> +

> +   The GNU C Library is distributed in the hope that it will be useful,

> +   but WITHOUT ANY WARRANTY; without even the implied warranty of

> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

> +   Lesser General Public License for more details.

> +

> +   You should have received a copy of the GNU Lesser General Public

> +   License along with the GNU C Library; if not, see

> +   <https://www.gnu.org/licenses/>.  */

> +

> +#include <sysdep.h>

> +

> +/* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5]));

> +   Returns 's'.  */

> +

> +#ifndef MEMSET

> +# define MEMSET memset

> +#endif

> +

> +       .machine  power9

> +ENTRY_TOCLESS (MEMSET, 5)

> +       CALL_MCOUNT 3

> +

Ok.
> +L(_memset):

> +       /* Assume memset of zero length is uncommon, and just let it go

> +          through the small path below.  */

> +       cmpldi  r5,64

> +

> +       /* Replicate byte to quad word.  */

> +       mtvsrws v0+32,r4

> +       vspltb  v0,v0,15

> +

> +       li      r7,16

> +       sldi    r8,r7,56

> +

> +       bgt     L(large)

> +

> +       /* For short lengths we want to avoid as many branches as possible.

> +          We use store VSX vector with length instructions to do this.  */

> +       sldi    r5,r5,56

> +

> +       addi    r10,r3,16

> +

> +       sub.    r11,r5,r8

> +       isellt  r11,0,r11       /* Saturate the subtraction to zero.  */

> +

> +       stxvl   v0+32,r3,r5

> +       stxvl   v0+32,r10,r11

> +

> +       addi    r9,r3,32

> +       addi    r10,r3,48

> +

> +       sub.    r11,r11,r8

> +       isellt  r11,0,r11

> +

> +       sub.    r5,r11,r8

> +       isellt  r5,0,r5

> +

> +       stxvl   v0+32,r9,r11

> +       stxvl   v0+32,r10,r5

> +

> +       blr

> +

Ok.

> +       .balign 16

> +L(large):

> +       mr      r6,r3   /* Don't modify r3 since we need to return it.  */

> +

> +       /* Get dest 16B aligned.  */

> +       neg     r0,r3

> +       clrldi. r7,r0,(64-4)

> +       beq     L(aligned)

> +       rldic   r9,r0,56,4      /* (~X & 0xf)<<56 "clrlsldi r9,r0,64-4,56".  */

> +

> +       stxvl   v0+32,r6,r9     /* Store up to 15B until aligned address.  */

> +

> +       add     r6,r6,r7

> +       sub     r5,r5,r7

> +

Ok.

> +       /* After alignment, if there is 127B or less left

s/127B/64B/

> +          go directly to the tail.  */

> +       cmpldi  r5,64

> +       blt     L(tail_64)

> +

> +       .balign 16

> +L(aligned):

> +       srdi.   r0,r5,7

> +       beq     L(tail_128)

> +

> +       cmpldi  cr5,r5,255

> +       cmpldi  cr6,r4,0

> +       crand   27,26,21

> +       bt      27,L(dcbz)

Maybe add a comment to explain this branch.
> +

> +       mtctr   r0

> +

> +       .balign 32

> +L(loop):

> +       stxv    v0+32,0(r6)

> +       stxv    v0+32,16(r6)

> +       stxv    v0+32,32(r6)

> +       stxv    v0+32,48(r6)

> +       stxv    v0+32,64(r6)

> +       stxv    v0+32,80(r6)

> +       stxv    v0+32,96(r6)

> +       stxv    v0+32,112(r6)

> +       addi    r6,r6,128

> +       bdnz    L(loop)

> +

Ok.

> +       .balign 16

> +L(tail):

> +       /* 127B or less left, finish the tail or return.  */

> +       andi.   r5,r5,127

> +       beqlr

> +

> +       cmpldi  r5,64

> +       blt     L(tail_64)

> +

> +       .balign 16

> +L(tail_128):

The label tail_128 made me think that here would be copied 128 bytes.
Maybe add a comment here.

> +       stxv    v0+32,0(r6)

> +       stxv    v0+32,16(r6)

> +       stxv    v0+32,32(r6)

> +       stxv    v0+32,48(r6)

> +       addi    r6,r6,64

> +       andi.   r5,r5,63

> +       beqlr

> +

> +       .balign 16

> +L(tail_64):

Maybe add a comment here to explay this section as well.

> +       sldi    r5,r5,56

> +

> +       addi    r10,r6,16

> +

> +       sub.    r11,r5,r8

> +       isellt  r11,0,r11

> +

> +       stxvl   v0+32,r6,r5

> +       stxvl   v0+32,r10,r11

> +

> +       sub.    r11,r11,r8

> +       blelr

> +

> +       addi    r9,r6,32

> +       addi    r10,r6,48

> +

> +       isellt  r11,0,r11

> +

> +       sub.    r5,r11,r8

> +       isellt  r5,0,r5

> +

> +       stxvl   v0+32,r9,r11

> +       stxvl   v0+32,r10,r5

> +

> +       blr

> +

Ok.

> +       .balign 16

> +L(dcbz):

> +       /* Special case when value is 0 and we have a long length to deal

> +          with.  Use dcbz to zero out a full cacheline of 128 bytes at a time.

> +          Before using dcbz though, we need to get the destination 128-byte

> +          aligned.  */

> +       neg     r0,r6

> +       clrldi. r0,r0,(64-7)t

> +       beq     L(dcbz_aligned)

> +

> +       sub     r5,r5,r0

> +       mtocrf  0x2,r0  /* These are the bits 57..59, the ones for sizes 64,

> +                          32 and 16 which are those that need to be check.  */

> +

Ok.

> +       /* Write 16~128 bytes until DST is aligned to 128 bytes.  */

> +64:    bf      25,32f

> +       stxv    v0+32,0(r6)

> +       stxv    v0+32,16(r6)

> +       stxv    v0+32,32(r6)

> +       stxv    v0+32,48(r6)

> +       addi    r6,r6,64

> +

> +32:    bf      26,16f

> +       stxv    v0+32,0(r6)

> +       stxv    v0+32,16(r6)

> +       addi    r6,r6,32

> +

> +16:    bf      27,L(dcbz_aligned)

> +       stxv    v0+32,0(r6)

> +       addi    r6,r6,16

> +

Ok.

> +       .balign 16

> +L(dcbz_aligned):

> +       /* Setup dcbz unroll offsets and count numbers.  */

> +       srdi.   r0,r5,9

> +       li      r9,128

> +       beq     L(bcdz_tail)

> +       li      r10,256

> +       li      r11,384

> +       mtctr   r0

> +

Ok.

> +       .balign 16

> +L(dcbz_loop):

> +       /* Sets 512 bytes to zero in each iteration, the loop unrolling shows

> +          a throughput boost for large sizes (2048 bytes or higher).  */

> +       dcbz    0,r6

> +       dcbz    r9,r6

> +       dcbz    r10,r6

> +       dcbz    r11,r6

> +       addi    r6,r6,512

> +       bdnz    L(dcbz_loop)

> +

> +       andi.   r5,r5,511

> +       beqlr

> +

Ok.

> +       .balign 16

> +L(bcdz_tail):

> +       /* We have 1~511 bytes remaining.  */

> +       srdi.   r0,r5,7

> +       beq     L(tail)

> +

> +       mtocrf  0x1,r0

> +

> +256:   bf      30,128f

> +       dcbz    0,r6

> +       dcbz    r9,r6

> +       addi    r6,r6,256

> +

> +128:   bf      31,L(tail)

> +       dcbz    0,r6

> +       addi    r6,r6,128

> +

> +       b       L(tail)

> +

Ok.

> +END_GEN_TB (MEMSET,TB_TOCLESS)

> +libc_hidden_builtin_def (memset)

> +

> +/* Copied from bzero.S to prevent the linker from inserting a stub

> +   between bzero and memset.  */

> +ENTRY_TOCLESS (__bzero)

> +       CALL_MCOUNT 3

> +       mr      r5,r4

> +       li      r4,0

> +       b       L(_memset)

> +END (__bzero)

> +#ifndef __bzero

> +weak_alias (__bzero, bzero)

> +#endif

Ok.

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile

> index 8aa46a370270..147ed42b218e 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/Makefile

> +++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile

> @@ -32,7 +32,8 @@ sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \

>                    strncase-power8

>  

>  ifneq (,$(filter %le,$(config-machine)))

> -sysdep_routines += strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \

> +sysdep_routines += memset-power10 \

> +                  strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \

>                    rawmemchr-power9 strlen-power9 strncpy-power9 stpncpy-power9 \

>                    strlen-power10

>  endif

Ok.

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/bzero.c b/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> index c3f819ff48d6..50a5320c6650 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> @@ -27,8 +27,16 @@ extern __typeof (bzero) __bzero_power4 attribute_hidden;

>  extern __typeof (bzero) __bzero_power6 attribute_hidden;

>  extern __typeof (bzero) __bzero_power7 attribute_hidden;

>  extern __typeof (bzero) __bzero_power8 attribute_hidden;

> +# ifdef __LITTLE_ENDIAN__

> +extern __typeof (bzero) __bzero_power10 attribute_hidden;

> +# endif

>  

>  libc_ifunc (__bzero,

> +# ifdef __LITTLE_ENDIAN__

> +           (hwcap2 & (PPC_FEATURE2_ARCH_3_1 | PPC_FEATURE2_HAS_ISEL)

> +            && hwcap & PPC_FEATURE_HAS_VSX)

> +           ? __bzero_power10 :

> +# endif

>              (hwcap2 & PPC_FEATURE2_ARCH_2_07)

>              ? __bzero_power8 :

>               (hwcap & PPC_FEATURE_HAS_VSX)

Ok.

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> index 1a6993616f2a..cd0d95ed9a94 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> @@ -73,6 +73,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

>  

>    /* Support sysdeps/powerpc/powerpc64/multiarch/memset.c.  */

>    IFUNC_IMPL (i, name, memset,

> +#ifdef __LITTLE_ENDIAN__

> +             IFUNC_IMPL_ADD (array, i, memset,

> +                             hwcap2 & (PPC_FEATURE2_ARCH_3_1 |

> +                                       PPC_FEATURE2_HAS_ISEL)

> +                             && hwcap & PPC_FEATURE_HAS_VSX,

> +                             __memset_power10)

> +#endif

>               IFUNC_IMPL_ADD (array, i, memset, hwcap2 & PPC_FEATURE2_ARCH_2_07,

>                               __memset_power8)

>               IFUNC_IMPL_ADD (array, i, memset, hwcap & PPC_FEATURE_HAS_VSX,

> @@ -174,6 +181,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

>  

>    /* Support sysdeps/powerpc/powerpc64/multiarch/bzero.c.  */

>    IFUNC_IMPL (i, name, bzero,

> +#ifdef __LITTLE_ENDIAN__

> +             IFUNC_IMPL_ADD (array, i, bzero,

> +                             hwcap2 & (PPC_FEATURE2_ARCH_3_1 |

> +                                       PPC_FEATURE2_HAS_ISEL)

> +                             && hwcap & PPC_FEATURE_HAS_VSX,

> +                             __bzero_power10)

> +#endif

>               IFUNC_IMPL_ADD (array, i, bzero, hwcap2 & PPC_FEATURE2_ARCH_2_07,

>                               __bzero_power8)

>               IFUNC_IMPL_ADD (array, i, bzero, hwcap & PPC_FEATURE_HAS_VSX,

Ok.

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> new file mode 100644

> index 000000000000..53a9535a2401

> --- /dev/null

> +++ b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> @@ -0,0 +1,27 @@

> +/* Optimized memset implementation for PowerPC64/POWER10.

> +   Copyright (C) 2021 Free Software Foundation, Inc.

> +   This file is part of the GNU C Library.

> +

> +   The GNU C Library is free software; you can redistribute it and/or

> +   modify it under the terms of the GNU Lesser General Public

> +   License as published by the Free Software Foundation; either

> +   version 2.1 of the License, or (at your option) any later version.

> +

> +   The GNU C Library is distributed in the hope that it will be useful,

> +   but WITHOUT ANY WARRANTY; without even the implied warranty of

> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

> +   Lesser General Public License for more details.

> +

> +   You should have received a copy of the GNU Lesser General Public

> +   License along with the GNU C Library; if not, see

> +   <https://www.gnu.org/licenses/>.  */

> +

> +#define MEMSET __memset_power10

> +

> +#undef libc_hidden_builtin_def

> +#define libc_hidden_builtin_def(name)

> +

> +#undef __bzero

> +#define __bzero __bzero_power10

> +

Ok.

> +#include <sysdeps/powerpc/powerpc64/le/power10/memset.S>

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset.c b/sysdeps/powerpc/powerpc64/multiarch/memset.c

> index d483f66f2744..6562646dffcf 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/memset.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/memset.c

> @@ -33,10 +33,18 @@ extern __typeof (__redirect_memset) __memset_power4 attribute_hidden;

>  extern __typeof (__redirect_memset) __memset_power6 attribute_hidden;

>  extern __typeof (__redirect_memset) __memset_power7 attribute_hidden;

>  extern __typeof (__redirect_memset) __memset_power8 attribute_hidden;

> +# ifdef __LITTLE_ENDIAN__

> +extern __typeof (__redirect_memset) __memset_power10 attribute_hidden;

> +# endif

>  

>  /* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle

>     ifunc symbol properly.  */

>  libc_ifunc (__libc_memset,

> +# ifdef __LITTLE_ENDIAN__

> +           (hwcap2 & (PPC_FEATURE2_ARCH_3_1 | PPC_FEATURE2_HAS_ISEL)

> +            && hwcap & PPC_FEATURE_HAS_VSX)

> +           ? __memset_power10 :

> +# endif

>              (hwcap2 & PPC_FEATURE2_ARCH_2_07)

>              ? __memset_power8 :

>               (hwcap & PPC_FEATURE_HAS_VSX)

Ok.

> -- 

> 2.26.2

>
Andreas Roeseler via Libc-alpha April 28, 2021, 8:28 p.m. | #2
Hi Raoni, this patch LGTM.

On 28/04/2021 11:40, Raoni Fassina Firmino via Libc-alpha wrote:
> This implementation is based on __memset_power8 and integrates a lot

> of suggestions from Anton Blanchard.

> 

> The biggest difference is that it makes extensive use of stxvl to

> alignment and tail code to avoid branches and small stores.  It has

> three main execution paths:

> 

> a) "Short lengths" for lengths up to 64 bytes, avoiding as many

>     branches as possible.

> 

> b) "General case" for larger lengths, it has an alignment section

>     using stxvl to avoid branches, a 128 bytes loop and then a tail

>     code, again using stxvl with few branches.

> 

> c) "Zeroing cache blocks" for lengths from 256 bytes upwards and set

>     value being zero.  It is mostly the __memset_power8 code but the

>     alignment phase was simplified because, at this point, address is

>     already 16-bytes aligned and also changed to use vector stores.

>     The tail code was also simplified to reuse the general case tail.

> 

> All unaligned stores use stxvl instructions that do not generate

> alignment interrupts on POWER10, making it safe to use on

> caching-inhibited memory.

> 

> On average, this implementation provides something around 30%

> improvement when compared to __memset_power8.

> ---

>   sysdeps/powerpc/powerpc64/le/power10/memset.S | 251 ++++++++++++++++++

>   sysdeps/powerpc/powerpc64/multiarch/Makefile  |   3 +-

>   sysdeps/powerpc/powerpc64/multiarch/bzero.c   |   8 +

>   .../powerpc64/multiarch/ifunc-impl-list.c     |  14 +

>   .../powerpc64/multiarch/memset-power10.S      |  27 ++

>   sysdeps/powerpc/powerpc64/multiarch/memset.c  |   8 +

>   6 files changed, 310 insertions(+), 1 deletion(-)

>   create mode 100644 sysdeps/powerpc/powerpc64/le/power10/memset.S

>   create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> 

> diff --git a/sysdeps/powerpc/powerpc64/le/power10/memset.S b/sysdeps/powerpc/powerpc64/le/power10/memset.S

> new file mode 100644

> index 000000000000..c8b77fc64596

> --- /dev/null

> +++ b/sysdeps/powerpc/powerpc64/le/power10/memset.S

> @@ -0,0 +1,251 @@

> +/* Optimized memset implementation for PowerPC64/POWER10.


Could be just POWER10.

> +   Copyright (C) 2021 Free Software Foundation, Inc.

> +   This file is part of the GNU C Library.

> +

> +   The GNU C Library is free software; you can redistribute it and/or

> +   modify it under the terms of the GNU Lesser General Public

> +   License as published by the Free Software Foundation; either

> +   version 2.1 of the License, or (at your option) any later version.

> +

> +   The GNU C Library is distributed in the hope that it will be useful,

> +   but WITHOUT ANY WARRANTY; without even the implied warranty of

> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

> +   Lesser General Public License for more details.

> +

> +   You should have received a copy of the GNU Lesser General Public

> +   License along with the GNU C Library; if not, see

> +   <https://www.gnu.org/licenses/>.  */

> +

> +#include <sysdep.h>

> +

> +/* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5]));

> +   Returns 's'.  */

> +

> +#ifndef MEMSET

> +# define MEMSET memset

> +#endif

> +

> +	.machine  power9

> +ENTRY_TOCLESS (MEMSET, 5)

> +	CALL_MCOUNT 3

> +

> +L(_memset):

> +	/* Assume memset of zero length is uncommon, and just let it go

> +	   through the small path below.  */

> +	cmpldi	r5,64

> +

> +	/* Replicate byte to quad word.  */

> +	mtvsrws v0+32,r4

> +	vspltb	v0,v0,15

> +

> +	li	r7,16

> +	sldi	r8,r7,56

> +

> +	bgt	L(large)

> +

> +	/* For short lengths we want to avoid as many branches as possible.

> +	   We use store VSX vector with length instructions to do this.  */

> +	sldi	r5,r5,56

> +

> +	addi	r10,r3,16

> +

> +	sub.	r11,r5,r8

> +	isellt	r11,0,r11	/* Saturate the subtraction to zero.  */

> +

> +	stxvl	v0+32,r3,r5

> +	stxvl	v0+32,r10,r11

> +

> +	addi	r9,r3,32

> +	addi	r10,r3,48

> +

> +	sub.	r11,r11,r8

> +	isellt	r11,0,r11

> +

> +	sub.	r5,r11,r8

> +	isellt	r5,0,r5

> +

> +	stxvl	v0+32,r9,r11

> +	stxvl	v0+32,r10,r5

> +

> +	blr

> +

> +	.balign	16

> +L(large):

> +	mr	r6,r3	/* Don't modify r3 since we need to return it.  */

> +

> +	/* Get dest 16B aligned.  */

> +	neg	r0,r3

> +	clrldi.	r7,r0,(64-4)

> +	beq	L(aligned)

> +	rldic	r9,r0,56,4	/* (~X & 0xf)<<56 "clrlsldi r9,r0,64-4,56".  */

> +

> +	stxvl	v0+32,r6,r9	/* Store up to 15B until aligned address.  */

> +

> +	add	r6,r6,r7

> +	sub	r5,r5,r7

> +

> +	/* After alignment, if there is 127B or less left

> +	   go directly to the tail.  */

> +	cmpldi	r5,64

> +	blt	L(tail_64)

> +

> +	.balign	16

> +L(aligned):

> +	srdi.	r0,r5,7

> +	beq	L(tail_128)

> +

> +	cmpldi	cr5,r5,255

> +	cmpldi	cr6,r4,0

> +	crand	27,26,21

> +	bt	27,L(dcbz)

> +

> +	mtctr	r0

> +

> +	.balign	32

> +L(loop):

> +	stxv	v0+32,0(r6)

> +	stxv	v0+32,16(r6)

> +	stxv	v0+32,32(r6)

> +	stxv	v0+32,48(r6)

> +	stxv	v0+32,64(r6)

> +	stxv	v0+32,80(r6)

> +	stxv	v0+32,96(r6)

> +	stxv	v0+32,112(r6)

> +	addi	r6,r6,128

> +	bdnz	L(loop)

> +

> +	.balign	16

> +L(tail):

> +	/* 127B or less left, finish the tail or return.  */

> +	andi.	r5,r5,127

> +	beqlr

> +

> +	cmpldi	r5,64

> +	blt	L(tail_64)

> +

> +	.balign	16

> +L(tail_128):

> +	stxv	v0+32,0(r6)

> +	stxv	v0+32,16(r6)

> +	stxv	v0+32,32(r6)

> +	stxv	v0+32,48(r6)

> +	addi	r6,r6,64

> +	andi.	r5,r5,63

> +	beqlr

> +

> +	.balign	16

> +L(tail_64):

> +	sldi	r5,r5,56

> +

> +	addi	r10,r6,16

> +

> +	sub.	r11,r5,r8

> +	isellt	r11,0,r11

> +

> +	stxvl	v0+32,r6,r5

> +	stxvl	v0+32,r10,r11

> +

> +	sub.	r11,r11,r8

> +	blelr

> +

> +	addi	r9,r6,32

> +	addi	r10,r6,48

> +

> +	isellt	r11,0,r11

> +

> +	sub.	r5,r11,r8

> +	isellt	r5,0,r5

> +

> +	stxvl	v0+32,r9,r11

> +	stxvl	v0+32,r10,r5

> +

> +	blr

> +

> +	.balign	16

> +L(dcbz):

> +	/* Special case when value is 0 and we have a long length to deal

> +	   with.  Use dcbz to zero out a full cacheline of 128 bytes at a time.

> +	   Before using dcbz though, we need to get the destination 128-byte

> +	   aligned.  */

> +	neg	r0,r6

> +	clrldi.	r0,r0,(64-7)

> +	beq	L(dcbz_aligned)

> +

> +	sub	r5,r5,r0

> +	mtocrf	0x2,r0	/* These are the bits 57..59, the ones for sizes 64,

> +			   32 and 16 which are those that need to be check.  */

> +

> +	/* Write 16~128 bytes until DST is aligned to 128 bytes.  */

> +64:	bf	25,32f

> +	stxv	v0+32,0(r6)

> +	stxv	v0+32,16(r6)

> +	stxv	v0+32,32(r6)

> +	stxv	v0+32,48(r6)

> +	addi	r6,r6,64

> +

> +32:	bf	26,16f

> +	stxv	v0+32,0(r6)

> +	stxv	v0+32,16(r6)

> +	addi	r6,r6,32

> +

> +16:	bf	27,L(dcbz_aligned)

> +	stxv	v0+32,0(r6)

> +	addi	r6,r6,16

> +

> +	.balign	16

> +L(dcbz_aligned):

> +	/* Setup dcbz unroll offsets and count numbers.  */

> +	srdi.	r0,r5,9

> +	li	r9,128

> +	beq	L(bcdz_tail)

> +	li	r10,256

> +	li	r11,384

> +	mtctr	r0

> +

> +	.balign	16

> +L(dcbz_loop):

> +	/* Sets 512 bytes to zero in each iteration, the loop unrolling shows

> +	   a throughput boost for large sizes (2048 bytes or higher).  */

> +	dcbz	0,r6

> +	dcbz	r9,r6

> +	dcbz	r10,r6

> +	dcbz	r11,r6

> +	addi	r6,r6,512

> +	bdnz	L(dcbz_loop)

> +

> +	andi.	r5,r5,511

> +	beqlr

> +

> +	.balign	16

> +L(bcdz_tail):

> +	/* We have 1~511 bytes remaining.  */

> +	srdi.	r0,r5,7

> +	beq	L(tail)

> +

> +	mtocrf	0x1,r0

> +

> +256:	bf	30,128f

> +	dcbz	0,r6

> +	dcbz	r9,r6

> +	addi	r6,r6,256

> +

> +128:	bf	31,L(tail)

> +	dcbz	0,r6

> +	addi	r6,r6,128

> +

> +	b	L(tail)

> +

> +END_GEN_TB (MEMSET,TB_TOCLESS)

> +libc_hidden_builtin_def (memset)

> +

> +/* Copied from bzero.S to prevent the linker from inserting a stub

> +   between bzero and memset.  */

> +ENTRY_TOCLESS (__bzero)

> +	CALL_MCOUNT 3

> +	mr	r5,r4

> +	li	r4,0

> +	b	L(_memset)

> +END (__bzero)

> +#ifndef __bzero

> +weak_alias (__bzero, bzero)

> +#endif

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile

> index 8aa46a370270..147ed42b218e 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/Makefile

> +++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile

> @@ -32,7 +32,8 @@ sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \

>   		   strncase-power8

> 

>   ifneq (,$(filter %le,$(config-machine)))

> -sysdep_routines += strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \

> +sysdep_routines += memset-power10 \

> +		   strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \

>   		   rawmemchr-power9 strlen-power9 strncpy-power9 stpncpy-power9 \

>   		   strlen-power10

>   endif

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/bzero.c b/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> index c3f819ff48d6..50a5320c6650 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> @@ -27,8 +27,16 @@ extern __typeof (bzero) __bzero_power4 attribute_hidden;

>   extern __typeof (bzero) __bzero_power6 attribute_hidden;

>   extern __typeof (bzero) __bzero_power7 attribute_hidden;

>   extern __typeof (bzero) __bzero_power8 attribute_hidden;

> +# ifdef __LITTLE_ENDIAN__

> +extern __typeof (bzero) __bzero_power10 attribute_hidden;

> +# endif

> 

>   libc_ifunc (__bzero,

> +# ifdef __LITTLE_ENDIAN__

> +	    (hwcap2 & (PPC_FEATURE2_ARCH_3_1 | PPC_FEATURE2_HAS_ISEL)

> +	     && hwcap & PPC_FEATURE_HAS_VSX)

> +	    ? __bzero_power10 :

> +# endif

>               (hwcap2 & PPC_FEATURE2_ARCH_2_07)

>               ? __bzero_power8 :

>   	      (hwcap & PPC_FEATURE_HAS_VSX)

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> index 1a6993616f2a..cd0d95ed9a94 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> @@ -73,6 +73,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

> 

>     /* Support sysdeps/powerpc/powerpc64/multiarch/memset.c.  */

>     IFUNC_IMPL (i, name, memset,

> +#ifdef __LITTLE_ENDIAN__

> +	      IFUNC_IMPL_ADD (array, i, memset,

> +			      hwcap2 & (PPC_FEATURE2_ARCH_3_1 |

> +					PPC_FEATURE2_HAS_ISEL)

> +			      && hwcap & PPC_FEATURE_HAS_VSX,

> +			      __memset_power10)

> +#endif

>   	      IFUNC_IMPL_ADD (array, i, memset, hwcap2 & PPC_FEATURE2_ARCH_2_07,

>   			      __memset_power8)

>   	      IFUNC_IMPL_ADD (array, i, memset, hwcap & PPC_FEATURE_HAS_VSX,

> @@ -174,6 +181,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

> 

>     /* Support sysdeps/powerpc/powerpc64/multiarch/bzero.c.  */

>     IFUNC_IMPL (i, name, bzero,

> +#ifdef __LITTLE_ENDIAN__

> +	      IFUNC_IMPL_ADD (array, i, bzero,

> +			      hwcap2 & (PPC_FEATURE2_ARCH_3_1 |

> +					PPC_FEATURE2_HAS_ISEL)

> +			      && hwcap & PPC_FEATURE_HAS_VSX,

> +			      __bzero_power10)

> +#endif

>   	      IFUNC_IMPL_ADD (array, i, bzero, hwcap2 & PPC_FEATURE2_ARCH_2_07,

>   			      __bzero_power8)

>   	      IFUNC_IMPL_ADD (array, i, bzero, hwcap & PPC_FEATURE_HAS_VSX,

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> new file mode 100644

> index 000000000000..53a9535a2401

> --- /dev/null

> +++ b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> @@ -0,0 +1,27 @@

> +/* Optimized memset implementation for PowerPC64/POWER10.

> +   Copyright (C) 2021 Free Software Foundation, Inc.

> +   This file is part of the GNU C Library.

> +

> +   The GNU C Library is free software; you can redistribute it and/or

> +   modify it under the terms of the GNU Lesser General Public

> +   License as published by the Free Software Foundation; either

> +   version 2.1 of the License, or (at your option) any later version.

> +

> +   The GNU C Library is distributed in the hope that it will be useful,

> +   but WITHOUT ANY WARRANTY; without even the implied warranty of

> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

> +   Lesser General Public License for more details.

> +

> +   You should have received a copy of the GNU Lesser General Public

> +   License along with the GNU C Library; if not, see

> +   <https://www.gnu.org/licenses/>.  */

> +

> +#define MEMSET __memset_power10

> +

> +#undef libc_hidden_builtin_def

> +#define libc_hidden_builtin_def(name)

> +

> +#undef __bzero

> +#define __bzero __bzero_power10

> +

> +#include <sysdeps/powerpc/powerpc64/le/power10/memset.S>

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset.c b/sysdeps/powerpc/powerpc64/multiarch/memset.c

> index d483f66f2744..6562646dffcf 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/memset.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/memset.c

> @@ -33,10 +33,18 @@ extern __typeof (__redirect_memset) __memset_power4 attribute_hidden;

>   extern __typeof (__redirect_memset) __memset_power6 attribute_hidden;

>   extern __typeof (__redirect_memset) __memset_power7 attribute_hidden;

>   extern __typeof (__redirect_memset) __memset_power8 attribute_hidden;

> +# ifdef __LITTLE_ENDIAN__

> +extern __typeof (__redirect_memset) __memset_power10 attribute_hidden;

> +# endif

> 

>   /* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle

>      ifunc symbol properly.  */

>   libc_ifunc (__libc_memset,

> +# ifdef __LITTLE_ENDIAN__

> +	    (hwcap2 & (PPC_FEATURE2_ARCH_3_1 | PPC_FEATURE2_HAS_ISEL)

> +	     && hwcap & PPC_FEATURE_HAS_VSX)

> +	    ? __memset_power10 :

> +# endif

>               (hwcap2 & PPC_FEATURE2_ARCH_2_07)

>               ? __memset_power8 :

>   	      (hwcap & PPC_FEATURE_HAS_VSX)

> 


Thanks,
-- 
Raphael Moreira Zinsly
Andreas Roeseler via Libc-alpha April 28, 2021, 8:49 p.m. | #3
Ok, no tests failing because of this change.

The logic LGTM, I just have some suggestions on comments and style.

Reviewed-by: Matheus Castanho <msc@linux.ibm.com>


Thanks!

Raoni Fassina Firmino via Libc-alpha <libc-alpha@sourceware.org> writes:

> This implementation is based on __memset_power8 and integrates a lot

> of suggestions from Anton Blanchard.

>

> The biggest difference is that it makes extensive use of stxvl to

> alignment and tail code to avoid branches and small stores.  It has

> three main execution paths:

>

> a) "Short lengths" for lengths up to 64 bytes, avoiding as many

>    branches as possible.

>

> b) "General case" for larger lengths, it has an alignment section

>    using stxvl to avoid branches, a 128 bytes loop and then a tail

>    code, again using stxvl with few branches.

>

> c) "Zeroing cache blocks" for lengths from 256 bytes upwards and set

>    value being zero.  It is mostly the __memset_power8 code but the

>    alignment phase was simplified because, at this point, address is

>    already 16-bytes aligned and also changed to use vector stores.

>    The tail code was also simplified to reuse the general case tail.

>

> All unaligned stores use stxvl instructions that do not generate

> alignment interrupts on POWER10, making it safe to use on

> caching-inhibited memory.

>

> On average, this implementation provides something around 30%

> improvement when compared to __memset_power8.

> ---

>  sysdeps/powerpc/powerpc64/le/power10/memset.S | 251 ++++++++++++++++++

>  sysdeps/powerpc/powerpc64/multiarch/Makefile  |   3 +-

>  sysdeps/powerpc/powerpc64/multiarch/bzero.c   |   8 +

>  .../powerpc64/multiarch/ifunc-impl-list.c     |  14 +

>  .../powerpc64/multiarch/memset-power10.S      |  27 ++

>  sysdeps/powerpc/powerpc64/multiarch/memset.c  |   8 +

>  6 files changed, 310 insertions(+), 1 deletion(-)

>  create mode 100644 sysdeps/powerpc/powerpc64/le/power10/memset.S

>  create mode 100644 sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

>

> diff --git a/sysdeps/powerpc/powerpc64/le/power10/memset.S b/sysdeps/powerpc/powerpc64/le/power10/memset.S

> new file mode 100644

> index 000000000000..c8b77fc64596

> --- /dev/null

> +++ b/sysdeps/powerpc/powerpc64/le/power10/memset.S

> @@ -0,0 +1,251 @@

> +/* Optimized memset implementation for PowerPC64/POWER10.

> +   Copyright (C) 2021 Free Software Foundation, Inc.

> +   This file is part of the GNU C Library.

> +

> +   The GNU C Library is free software; you can redistribute it and/or

> +   modify it under the terms of the GNU Lesser General Public

> +   License as published by the Free Software Foundation; either

> +   version 2.1 of the License, or (at your option) any later version.

> +

> +   The GNU C Library is distributed in the hope that it will be useful,

> +   but WITHOUT ANY WARRANTY; without even the implied warranty of

> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

> +   Lesser General Public License for more details.

> +

> +   You should have received a copy of the GNU Lesser General Public

> +   License along with the GNU C Library; if not, see

> +   <https://www.gnu.org/licenses/>.  */

> +

> +#include <sysdep.h>

> +

> +/* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5]));

> +   Returns 's'.  */

> +

> +#ifndef MEMSET

> +# define MEMSET memset

> +#endif

> +

> +	.machine  power9

> +ENTRY_TOCLESS (MEMSET, 5)

> +	CALL_MCOUNT 3

> +

> +L(_memset):

> +	/* Assume memset of zero length is uncommon, and just let it go

> +	   through the small path below.  */

> +	cmpldi	r5,64

> +

> +	/* Replicate byte to quad word.  */

> +	mtvsrws v0+32,r4

> +	vspltb	v0,v0,15


Why not simply use mtvsrd here? The byte splat part will have to be done
separately anyways:

      mtvsrd v0+32,r4
      vspltb v0,v0,7

> +

> +	li	r7,16

> +	sldi	r8,r7,56

> +

> +	bgt	L(large)

> +

> +	/* For short lengths we want to avoid as many branches as possible.

> +	   We use store VSX vector with length instructions to do this.  */

> +	sldi	r5,r5,56

> +

> +	addi	r10,r3,16

> +

> +	sub.	r11,r5,r8

> +	isellt	r11,0,r11	/* Saturate the subtraction to zero.  */

> +

> +	stxvl	v0+32,r3,r5

> +	stxvl	v0+32,r10,r11

> +

> +	addi	r9,r3,32

> +	addi	r10,r3,48

> +

> +	sub.	r11,r11,r8

> +	isellt	r11,0,r11

> +

> +	sub.	r5,r11,r8

> +	isellt	r5,0,r5


Minor detail: I see this construct appears many times in the code. You
could create a macro for it like:

#define SUBS(rt,ra,rb)   \
        sub.   rt,ra,rb; \
        isellt rt,0,rt;

> +

> +	stxvl	v0+32,r9,r11

> +	stxvl	v0+32,r10,r5

> +

> +	blr


Ok. Takes advantage of the fact that if the length passed to stxvl is
zero, nothing is done.

Maybe you could expand the comment at the beginning of this block to
make that trick clear.

> +

> +	.balign	16

> +L(large):

> +	mr	r6,r3	/* Don't modify r3 since we need to return it.  */

> +

> +	/* Get dest 16B aligned.  */

> +	neg	r0,r3

> +	clrldi.	r7,r0,(64-4)

> +	beq	L(aligned)

> +	rldic	r9,r0,56,4	/* (~X & 0xf)<<56 "clrlsldi r9,r0,64-4,56".  */


Why not just use clrlsldi as noted in the comment? It makes clearer what
the instruction is doing.

> +

> +	stxvl	v0+32,r6,r9	/* Store up to 15B until aligned address.  */

> +

> +	add	r6,r6,r7

> +	sub	r5,r5,r7


Ok. Store and update pointers by number of bytes left to 16B-aligned.

> +

> +	/* After alignment, if there is 127B or less left

> +	   go directly to the tail.  */


127B -> 63B

> +	cmpldi	r5,64

> +	blt	L(tail_64)

> +

> +	.balign	16

> +L(aligned):

> +	srdi.	r0,r5,7

> +	beq	L(tail_128)

> +

> +	cmpldi	cr5,r5,255

> +	cmpldi	cr6,r4,0

> +	crand	27,26,21

> +	bt	27,L(dcbz)


This last block deserves a comment, as it is really hard to follow.

IIUC, this is what this code is checking:

     if r5 > 255 && r4 == 0
        goto L(dcbz)

So if we have at least 256B left and we are setting zeroes, then use the
dcbz strategy. Ok.

> +

> +	mtctr	r0

> +

> +	.balign	32

> +L(loop):

> +	stxv	v0+32,0(r6)

> +	stxv	v0+32,16(r6)

> +	stxv	v0+32,32(r6)

> +	stxv	v0+32,48(r6)

> +	stxv	v0+32,64(r6)

> +	stxv	v0+32,80(r6)

> +	stxv	v0+32,96(r6)

> +	stxv	v0+32,112(r6)

> +	addi	r6,r6,128

> +	bdnz	L(loop)

> +


Ok.

> +	.balign	16

> +L(tail):

> +	/* 127B or less left, finish the tail or return.  */

> +	andi.	r5,r5,127

> +	beqlr

> +

> +	cmpldi	r5,64

> +	blt	L(tail_64)

> +


Ok. If we have less than 64, skip the 64B copy below.

> +	.balign	16

> +L(tail_128):

> +	stxv	v0+32,0(r6)

> +	stxv	v0+32,16(r6)

> +	stxv	v0+32,32(r6)

> +	stxv	v0+32,48(r6)

> +	addi	r6,r6,64

> +	andi.	r5,r5,63

> +	beqlr

> +


Ok. Copy 64B.

> +	.balign	16

> +L(tail_64):

> +	sldi	r5,r5,56

> +

> +	addi	r10,r6,16

> +

> +	sub.	r11,r5,r8

> +	isellt	r11,0,r11

> +

> +	stxvl	v0+32,r6,r5

> +	stxvl	v0+32,r10,r11

> +

> +	sub.	r11,r11,r8

> +	blelr

> +

> +	addi	r9,r6,32

> +	addi	r10,r6,48

> +

> +	isellt	r11,0,r11

> +

> +	sub.	r5,r11,r8

> +	isellt	r5,0,r5

> +

> +	stxvl	v0+32,r9,r11

> +	stxvl	v0+32,r10,r5

> +

> +	blr

> +


Ok.

> +	.balign	16

> +L(dcbz):

> +	/* Special case when value is 0 and we have a long length to deal

> +	   with.  Use dcbz to zero out a full cacheline of 128 bytes at a time.

> +	   Before using dcbz though, we need to get the destination 128-byte

> +	   aligned.  */

> +	neg	r0,r6

> +	clrldi.	r0,r0,(64-7)

> +	beq	L(dcbz_aligned)

> +

> +	sub	r5,r5,r0

> +	mtocrf	0x2,r0	/* These are the bits 57..59, the ones for sizes 64,

> +			   32 and 16 which are those that need to be check.  */


need to be check -> need to be checked

Please add to the comment that these bits are being set to cr6. mtocrf
is not one of the most straightfoward instructions to read =/

> +

> +	/* Write 16~128 bytes until DST is aligned to 128 bytes.  */


16~128 -> 16-128

> +64:	bf	25,32f

> +	stxv	v0+32,0(r6)

> +	stxv	v0+32,16(r6)

> +	stxv	v0+32,32(r6)

> +	stxv	v0+32,48(r6)

> +	addi	r6,r6,64

> +


Ok. Store 64 if len >= 64.

> +32:	bf	26,16f

> +	stxv	v0+32,0(r6)

> +	stxv	v0+32,16(r6)

> +	addi	r6,r6,32

> +


Ok. Store 32 if len >= 32.

> +16:	bf	27,L(dcbz_aligned)

> +	stxv	v0+32,0(r6)

> +	addi	r6,r6,16

> +


Ok. Store 16 if len >= 16.

> +	.balign	16

> +L(dcbz_aligned):

> +	/* Setup dcbz unroll offsets and count numbers.  */

> +	srdi.	r0,r5,9

> +	li	r9,128

> +	beq	L(bcdz_tail)

> +	li	r10,256

> +	li	r11,384

> +	mtctr	r0

> +


Ok.

> +	.balign	16

> +L(dcbz_loop):

> +	/* Sets 512 bytes to zero in each iteration, the loop unrolling shows

> +	   a throughput boost for large sizes (2048 bytes or higher).  */

> +	dcbz	0,r6

> +	dcbz	r9,r6

> +	dcbz	r10,r6

> +	dcbz	r11,r6

> +	addi	r6,r6,512

> +	bdnz	L(dcbz_loop)

> +

> +	andi.	r5,r5,511

> +	beqlr

> +


Ok.

> +	.balign	16

> +L(bcdz_tail):

> +	/* We have 1~511 bytes remaining.  */


1~511 -> 1-511

> +	srdi.	r0,r5,7

> +	beq	L(tail)

> +

> +	mtocrf	0x1,r0

> +

> +256:	bf	30,128f

> +	dcbz	0,r6

> +	dcbz	r9,r6

> +	addi	r6,r6,256

> +


Ok.

> +128:	bf	31,L(tail)

> +	dcbz	0,r6

> +	addi	r6,r6,128

> +

> +	b	L(tail)

> +


Ok.

> +END_GEN_TB (MEMSET,TB_TOCLESS)

> +libc_hidden_builtin_def (memset)

> +

> +/* Copied from bzero.S to prevent the linker from inserting a stub

> +   between bzero and memset.  */

> +ENTRY_TOCLESS (__bzero)

> +	CALL_MCOUNT 3


Should this CALL_MCOUNT 2 since bzero receives just 2 args?

> +	mr	r5,r4

> +	li	r4,0

> +	b	L(_memset)

> +END (__bzero)

> +#ifndef __bzero

> +weak_alias (__bzero, bzero)

> +#endif

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile

> index 8aa46a370270..147ed42b218e 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/Makefile

> +++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile

> @@ -32,7 +32,8 @@ sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \

>  		   strncase-power8

>

>  ifneq (,$(filter %le,$(config-machine)))

> -sysdep_routines += strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \

> +sysdep_routines += memset-power10 \

> +		   strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \

>  		   rawmemchr-power9 strlen-power9 strncpy-power9 stpncpy-power9 \

>  		   strlen-power10

>  endif

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/bzero.c b/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> index c3f819ff48d6..50a5320c6650 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/bzero.c

> @@ -27,8 +27,16 @@ extern __typeof (bzero) __bzero_power4 attribute_hidden;

>  extern __typeof (bzero) __bzero_power6 attribute_hidden;

>  extern __typeof (bzero) __bzero_power7 attribute_hidden;

>  extern __typeof (bzero) __bzero_power8 attribute_hidden;

> +# ifdef __LITTLE_ENDIAN__

> +extern __typeof (bzero) __bzero_power10 attribute_hidden;

> +# endif

>

>  libc_ifunc (__bzero,

> +# ifdef __LITTLE_ENDIAN__

> +	    (hwcap2 & (PPC_FEATURE2_ARCH_3_1 | PPC_FEATURE2_HAS_ISEL)

> +	     && hwcap & PPC_FEATURE_HAS_VSX)

> +	    ? __bzero_power10 :

> +# endif

>              (hwcap2 & PPC_FEATURE2_ARCH_2_07)

>              ? __bzero_power8 :

>  	      (hwcap & PPC_FEATURE_HAS_VSX)

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> index 1a6993616f2a..cd0d95ed9a94 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c

> @@ -73,6 +73,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

>

>    /* Support sysdeps/powerpc/powerpc64/multiarch/memset.c.  */

>    IFUNC_IMPL (i, name, memset,

> +#ifdef __LITTLE_ENDIAN__

> +	      IFUNC_IMPL_ADD (array, i, memset,

> +			      hwcap2 & (PPC_FEATURE2_ARCH_3_1 |

> +					PPC_FEATURE2_HAS_ISEL)

> +			      && hwcap & PPC_FEATURE_HAS_VSX,

> +			      __memset_power10)

> +#endif

>  	      IFUNC_IMPL_ADD (array, i, memset, hwcap2 & PPC_FEATURE2_ARCH_2_07,

>  			      __memset_power8)

>  	      IFUNC_IMPL_ADD (array, i, memset, hwcap & PPC_FEATURE_HAS_VSX,

> @@ -174,6 +181,13 @@ __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,

>

>    /* Support sysdeps/powerpc/powerpc64/multiarch/bzero.c.  */

>    IFUNC_IMPL (i, name, bzero,

> +#ifdef __LITTLE_ENDIAN__

> +	      IFUNC_IMPL_ADD (array, i, bzero,

> +			      hwcap2 & (PPC_FEATURE2_ARCH_3_1 |

> +					PPC_FEATURE2_HAS_ISEL)

> +			      && hwcap & PPC_FEATURE_HAS_VSX,

> +			      __bzero_power10)

> +#endif

>  	      IFUNC_IMPL_ADD (array, i, bzero, hwcap2 & PPC_FEATURE2_ARCH_2_07,

>  			      __bzero_power8)

>  	      IFUNC_IMPL_ADD (array, i, bzero, hwcap & PPC_FEATURE_HAS_VSX,

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> new file mode 100644

> index 000000000000..53a9535a2401

> --- /dev/null

> +++ b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> @@ -0,0 +1,27 @@

> +/* Optimized memset implementation for PowerPC64/POWER10.

> +   Copyright (C) 2021 Free Software Foundation, Inc.

> +   This file is part of the GNU C Library.

> +

> +   The GNU C Library is free software; you can redistribute it and/or

> +   modify it under the terms of the GNU Lesser General Public

> +   License as published by the Free Software Foundation; either

> +   version 2.1 of the License, or (at your option) any later version.

> +

> +   The GNU C Library is distributed in the hope that it will be useful,

> +   but WITHOUT ANY WARRANTY; without even the implied warranty of

> +   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU

> +   Lesser General Public License for more details.

> +

> +   You should have received a copy of the GNU Lesser General Public

> +   License along with the GNU C Library; if not, see

> +   <https://www.gnu.org/licenses/>.  */

> +

> +#define MEMSET __memset_power10

> +

> +#undef libc_hidden_builtin_def

> +#define libc_hidden_builtin_def(name)

> +

> +#undef __bzero

> +#define __bzero __bzero_power10

> +

> +#include <sysdeps/powerpc/powerpc64/le/power10/memset.S>

> diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset.c b/sysdeps/powerpc/powerpc64/multiarch/memset.c

> index d483f66f2744..6562646dffcf 100644

> --- a/sysdeps/powerpc/powerpc64/multiarch/memset.c

> +++ b/sysdeps/powerpc/powerpc64/multiarch/memset.c

> @@ -33,10 +33,18 @@ extern __typeof (__redirect_memset) __memset_power4 attribute_hidden;

>  extern __typeof (__redirect_memset) __memset_power6 attribute_hidden;

>  extern __typeof (__redirect_memset) __memset_power7 attribute_hidden;

>  extern __typeof (__redirect_memset) __memset_power8 attribute_hidden;

> +# ifdef __LITTLE_ENDIAN__

> +extern __typeof (__redirect_memset) __memset_power10 attribute_hidden;

> +# endif

>

>  /* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle

>     ifunc symbol properly.  */

>  libc_ifunc (__libc_memset,

> +# ifdef __LITTLE_ENDIAN__

> +	    (hwcap2 & (PPC_FEATURE2_ARCH_3_1 | PPC_FEATURE2_HAS_ISEL)

> +	     && hwcap & PPC_FEATURE_HAS_VSX)

> +	    ? __memset_power10 :

> +# endif

>              (hwcap2 & PPC_FEATURE2_ARCH_2_07)

>              ? __memset_power8 :

>  	      (hwcap & PPC_FEATURE_HAS_VSX)



--
Matheus Castanho
Andreas Roeseler via Libc-alpha April 29, 2021, 6:39 p.m. | #4
Thanks for the review Raphael.

> > diff --git a/sysdeps/powerpc/powerpc64/le/power10/memset.S b/sysdeps/powerpc/powerpc64/le/power10/memset.S

> > new file mode 100644

> > index 000000000000..c8b77fc64596

> > --- /dev/null

> > +++ b/sysdeps/powerpc/powerpc64/le/power10/memset.S

> > @@ -0,0 +1,251 @@

> > +/* Optimized memset implementation for PowerPC64/POWER10.

> 

> Could be just POWER10.


Done. kind of.

I had missed that now there is a reference in the same folder :D. Used
"POWER10 LE.", hope that is ok.


> > diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> > new file mode 100644

> > index 000000000000..53a9535a2401

> > --- /dev/null

> > +++ b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S

> > @@ -0,0 +1,27 @@

> > +/* Optimized memset implementation for PowerPC64/POWER10.


Also changed here.


o/
Raoni
Andreas Roeseler via Libc-alpha April 29, 2021, 6:40 p.m. | #5
Thanks for the review Lucas, please let me know if I missed something.

On Wed, Apr 28, 2021 at 03:48:28PM -0300, Lucas A. M. Magalhaes wrote:
> > +       /* After alignment, if there is 127B or less left

> s/127B/64B/


Done.

This is an awkward position/block, this comment is about the whole
block, up until the `beq  L(tail_128)`, but there is the label
'L(aligned)' in the middle that makes it hard to underhand. But it truly
is that, after the alignment if there is less than 128 bytes is goes to
the tail, but there is this optimization to go straight to the part of
the tail depending on the amount left.

any way, refrased it to be a single line (less obstrusive) and add a new
one for the second branch:

> > +          go directly to the tail.  */

> > +       cmpldi  r5,64

> > +       blt     L(tail_64)

> > +

> > +       .balign 16

> > +L(aligned):

> > +       srdi.   r0,r5,7

> > +       beq     L(tail_128)


Here^, added another after L(aligned).


> > +

> > +       cmpldi  cr5,r5,255

> > +       cmpldi  cr6,r4,0

> > +       crand   27,26,21

> > +       bt      27,L(dcbz)

> Maybe add a comment to explain this branch.


Done.

I was counting on the comment on the label itself, but I guess it makes
sense to add a brief comment  here also, avoid going back and forward to
understand the condition check.


> > +       .balign 16

> > +L(tail_128):

> The label tail_128 made me think that here would be copied 128 bytes.

> Maybe add a comment here.


Done.

Sorry, yes, all this "tail_*" sections are "up to", the number being the
maximum that it will write. But this one is in fact from 64 up to 128.


> 

> > +       stxv    v0+32,0(r6)

> > +       stxv    v0+32,16(r6)

> > +       stxv    v0+32,32(r6)

> > +       stxv    v0+32,48(r6)

> > +       addi    r6,r6,64

> > +       andi.   r5,r5,63

> > +       beqlr

> > +

> > +       .balign 16

> > +L(tail_64):

> Maybe add a comment here to explay this section as well.


Done.


o/
Raoni
Andreas Roeseler via Libc-alpha April 29, 2021, 9:40 p.m. | #6
Thanks for the review Matheus, let me know if I missed something.

> > +L(_memset):

> > +	/* Assume memset of zero length is uncommon, and just let it go

> > +	   through the small path below.  */

> > +	cmpldi	r5,64

> > +

> > +	/* Replicate byte to quad word.  */

> > +	mtvsrws v0+32,r4

> > +	vspltb	v0,v0,15

> 

> Why not simply use mtvsrd here? The byte splat part will have to be done

> separately anyways:

> 

>       mtvsrd v0+32,r4

>       vspltb v0,v0,7


Done.

Maybe this one is more intuitive. Don't have to wonder if the previous
word splat.


> > +	sub.	r11,r5,r8

> > +	isellt	r11,0,r11	/* Saturate the subtraction to zero.  */

> > +

> > +	stxvl	v0+32,r3,r5

> > +	stxvl	v0+32,r10,r11

> > +

> > +	addi	r9,r3,32

> > +	addi	r10,r3,48

> > +

> > +	sub.	r11,r11,r8

> > +	isellt	r11,0,r11

> > +

> > +	sub.	r5,r11,r8

> > +	isellt	r5,0,r5

> 

> Minor detail: I see this construct appears many times in the code. You

> could create a macro for it like:

> 

> #define SUBS(rt,ra,rb)   \

>         sub.   rt,ra,rb; \

>         isellt rt,0,rt;


I had this before, but I got to the conclusion it was hindering seen all
the code. Since it is only two instructions I though the macro benefits
were marginal and added some noise to have macro and mnemonics mixed up
in the code. There was a practical reason though, in one case I have a
blelr in the tail, so the macro could not be use in this case anyway.

But is there is a strong opinion here I can change it.


> > +

> > +	stxvl	v0+32,r9,r11

> > +	stxvl	v0+32,r10,r5

> > +

> > +	blr

> 

> Ok. Takes advantage of the fact that if the length passed to stxvl is

> zero, nothing is done.

> 

> Maybe you could expand the comment at the beginning of this block to

> make that trick clear.


Done.

 
> > +

> > +	.balign	16

> > +L(large):

> > +	mr	r6,r3	/* Don't modify r3 since we need to return it.  */

> > +

> > +	/* Get dest 16B aligned.  */

> > +	neg	r0,r3

> > +	clrldi.	r7,r0,(64-4)

> > +	beq	L(aligned)

> > +	rldic	r9,r0,56,4	/* (~X & 0xf)<<56 "clrlsldi r9,r0,64-4,56".  */

> 

> Why not just use clrlsldi as noted in the comment? It makes clearer what

> the instruction is doing.


Because of this:

	clrldi.	r7,r0,(64-4)
	beq	L(aligned)
	clrlsldi	r9,r0,64-4,56

It is too big to stay in the same tab column ad the rest of the
instructions and the options to fix this weren't great. It was just a
formatting decision, hence the comment to try to help out a little.
Also, in this particular case rldic reading of the values is not that
terrible, it is like: Keep the 4 least significant bits, shift then 56
bits left.

I can change if you or anyone else feel strong about it.


> > +

> > +	/* After alignment, if there is 127B or less left

> > +	   go directly to the tail.  */

> 

> 127B -> 63B


Done.


> > +	cmpldi	r5,64

> > +	blt	L(tail_64)

> > +

> > +	.balign	16

> > +L(aligned):

> > +	srdi.	r0,r5,7

> > +	beq	L(tail_128)

> > +

> > +	cmpldi	cr5,r5,255

> > +	cmpldi	cr6,r4,0

> > +	crand	27,26,21

> > +	bt	27,L(dcbz)

> 

> This last block deserves a comment, as it is really hard to follow.

> 

> IIUC, this is what this code is checking:

> 

>      if r5 > 255 && r4 == 0

>         goto L(dcbz)

> 

> So if we have at least 256B left and we are setting zeroes, then use the

> dcbz strategy. Ok.


Done.

I did the comment in one line, but mostly the same, let me know if it
explain it well. I decided to use the arguments names instead of
registers in hope to be more meaningful, but lets see it is not more
confusing.

> > +	.balign	16

> > +L(dcbz):

> > +	/* Special case when value is 0 and we have a long length to deal

> > +	   with.  Use dcbz to zero out a full cacheline of 128 bytes at a time.

> > +	   Before using dcbz though, we need to get the destination 128-byte

> > +	   aligned.  */

> > +	neg	r0,r6

> > +	clrldi.	r0,r0,(64-7)

> > +	beq	L(dcbz_aligned)

> > +

> > +	sub	r5,r5,r0

> > +	mtocrf	0x2,r0	/* These are the bits 57..59, the ones for sizes 64,

> > +			   32 and 16 which are those that need to be check.  */

> 

> need to be check -> need to be checked

> 

> Please add to the comment that these bits are being set to cr6. mtocrf

> is not one of the most straightfoward instructions to read =/


Done.

> > +	/* Write 16~128 bytes until DST is aligned to 128 bytes.  */

> 

> 16~128 -> 16-128


Done.


> > +	.balign	16

> > +L(bcdz_tail):

> > +	/* We have 1~511 bytes remaining.  */

> 

> 1~511 -> 1-511


Done.


> > +END_GEN_TB (MEMSET,TB_TOCLESS)

> > +libc_hidden_builtin_def (memset)

> > +

> > +/* Copied from bzero.S to prevent the linker from inserting a stub

> > +   between bzero and memset.  */

> > +ENTRY_TOCLESS (__bzero)

> > +	CALL_MCOUNT 3

> 

> Should this CALL_MCOUNT 2 since bzero receives just 2 args?


Done.

You are right here, good catch.


o/
Raoni

Patch

diff --git a/sysdeps/powerpc/powerpc64/le/power10/memset.S b/sysdeps/powerpc/powerpc64/le/power10/memset.S
new file mode 100644
index 000000000000..c8b77fc64596
--- /dev/null
+++ b/sysdeps/powerpc/powerpc64/le/power10/memset.S
@@ -0,0 +1,251 @@ 
+/* Optimized memset implementation for PowerPC64/POWER10.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#include <sysdep.h>
+
+/* void * [r3] memset (void *s [r3], int c [r4], size_t n [r5]));
+   Returns 's'.  */
+
+#ifndef MEMSET
+# define MEMSET memset
+#endif
+
+	.machine  power9
+ENTRY_TOCLESS (MEMSET, 5)
+	CALL_MCOUNT 3
+
+L(_memset):
+	/* Assume memset of zero length is uncommon, and just let it go
+	   through the small path below.  */
+	cmpldi	r5,64
+
+	/* Replicate byte to quad word.  */
+	mtvsrws v0+32,r4
+	vspltb	v0,v0,15
+
+	li	r7,16
+	sldi	r8,r7,56
+
+	bgt	L(large)
+
+	/* For short lengths we want to avoid as many branches as possible.
+	   We use store VSX vector with length instructions to do this.  */
+	sldi	r5,r5,56
+
+	addi	r10,r3,16
+
+	sub.	r11,r5,r8
+	isellt	r11,0,r11	/* Saturate the subtraction to zero.  */
+
+	stxvl	v0+32,r3,r5
+	stxvl	v0+32,r10,r11
+
+	addi	r9,r3,32
+	addi	r10,r3,48
+
+	sub.	r11,r11,r8
+	isellt	r11,0,r11
+
+	sub.	r5,r11,r8
+	isellt	r5,0,r5
+
+	stxvl	v0+32,r9,r11
+	stxvl	v0+32,r10,r5
+
+	blr
+
+	.balign	16
+L(large):
+	mr	r6,r3	/* Don't modify r3 since we need to return it.  */
+
+	/* Get dest 16B aligned.  */
+	neg	r0,r3
+	clrldi.	r7,r0,(64-4)
+	beq	L(aligned)
+	rldic	r9,r0,56,4	/* (~X & 0xf)<<56 "clrlsldi r9,r0,64-4,56".  */
+
+	stxvl	v0+32,r6,r9	/* Store up to 15B until aligned address.  */
+
+	add	r6,r6,r7
+	sub	r5,r5,r7
+
+	/* After alignment, if there is 127B or less left
+	   go directly to the tail.  */
+	cmpldi	r5,64
+	blt	L(tail_64)
+
+	.balign	16
+L(aligned):
+	srdi.	r0,r5,7
+	beq	L(tail_128)
+
+	cmpldi	cr5,r5,255
+	cmpldi	cr6,r4,0
+	crand	27,26,21
+	bt	27,L(dcbz)
+
+	mtctr	r0
+
+	.balign	32
+L(loop):
+	stxv	v0+32,0(r6)
+	stxv	v0+32,16(r6)
+	stxv	v0+32,32(r6)
+	stxv	v0+32,48(r6)
+	stxv	v0+32,64(r6)
+	stxv	v0+32,80(r6)
+	stxv	v0+32,96(r6)
+	stxv	v0+32,112(r6)
+	addi	r6,r6,128
+	bdnz	L(loop)
+
+	.balign	16
+L(tail):
+	/* 127B or less left, finish the tail or return.  */
+	andi.	r5,r5,127
+	beqlr
+
+	cmpldi	r5,64
+	blt	L(tail_64)
+
+	.balign	16
+L(tail_128):
+	stxv	v0+32,0(r6)
+	stxv	v0+32,16(r6)
+	stxv	v0+32,32(r6)
+	stxv	v0+32,48(r6)
+	addi	r6,r6,64
+	andi.	r5,r5,63
+	beqlr
+
+	.balign	16
+L(tail_64):
+	sldi	r5,r5,56
+
+	addi	r10,r6,16
+
+	sub.	r11,r5,r8
+	isellt	r11,0,r11
+
+	stxvl	v0+32,r6,r5
+	stxvl	v0+32,r10,r11
+
+	sub.	r11,r11,r8
+	blelr
+
+	addi	r9,r6,32
+	addi	r10,r6,48
+
+	isellt	r11,0,r11
+
+	sub.	r5,r11,r8
+	isellt	r5,0,r5
+
+	stxvl	v0+32,r9,r11
+	stxvl	v0+32,r10,r5
+
+	blr
+
+	.balign	16
+L(dcbz):
+	/* Special case when value is 0 and we have a long length to deal
+	   with.  Use dcbz to zero out a full cacheline of 128 bytes at a time.
+	   Before using dcbz though, we need to get the destination 128-byte
+	   aligned.  */
+	neg	r0,r6
+	clrldi.	r0,r0,(64-7)
+	beq	L(dcbz_aligned)
+
+	sub	r5,r5,r0
+	mtocrf	0x2,r0	/* These are the bits 57..59, the ones for sizes 64,
+			   32 and 16 which are those that need to be check.  */
+
+	/* Write 16~128 bytes until DST is aligned to 128 bytes.  */
+64:	bf	25,32f
+	stxv	v0+32,0(r6)
+	stxv	v0+32,16(r6)
+	stxv	v0+32,32(r6)
+	stxv	v0+32,48(r6)
+	addi	r6,r6,64
+
+32:	bf	26,16f
+	stxv	v0+32,0(r6)
+	stxv	v0+32,16(r6)
+	addi	r6,r6,32
+
+16:	bf	27,L(dcbz_aligned)
+	stxv	v0+32,0(r6)
+	addi	r6,r6,16
+
+	.balign	16
+L(dcbz_aligned):
+	/* Setup dcbz unroll offsets and count numbers.  */
+	srdi.	r0,r5,9
+	li	r9,128
+	beq	L(bcdz_tail)
+	li	r10,256
+	li	r11,384
+	mtctr	r0
+
+	.balign	16
+L(dcbz_loop):
+	/* Sets 512 bytes to zero in each iteration, the loop unrolling shows
+	   a throughput boost for large sizes (2048 bytes or higher).  */
+	dcbz	0,r6
+	dcbz	r9,r6
+	dcbz	r10,r6
+	dcbz	r11,r6
+	addi	r6,r6,512
+	bdnz	L(dcbz_loop)
+
+	andi.	r5,r5,511
+	beqlr
+
+	.balign	16
+L(bcdz_tail):
+	/* We have 1~511 bytes remaining.  */
+	srdi.	r0,r5,7
+	beq	L(tail)
+
+	mtocrf	0x1,r0
+
+256:	bf	30,128f
+	dcbz	0,r6
+	dcbz	r9,r6
+	addi	r6,r6,256
+
+128:	bf	31,L(tail)
+	dcbz	0,r6
+	addi	r6,r6,128
+
+	b	L(tail)
+
+END_GEN_TB (MEMSET,TB_TOCLESS)
+libc_hidden_builtin_def (memset)
+
+/* Copied from bzero.S to prevent the linker from inserting a stub
+   between bzero and memset.  */
+ENTRY_TOCLESS (__bzero)
+	CALL_MCOUNT 3
+	mr	r5,r4
+	li	r4,0
+	b	L(_memset)
+END (__bzero)
+#ifndef __bzero
+weak_alias (__bzero, bzero)
+#endif
diff --git a/sysdeps/powerpc/powerpc64/multiarch/Makefile b/sysdeps/powerpc/powerpc64/multiarch/Makefile
index 8aa46a370270..147ed42b218e 100644
--- a/sysdeps/powerpc/powerpc64/multiarch/Makefile
+++ b/sysdeps/powerpc/powerpc64/multiarch/Makefile
@@ -32,7 +32,8 @@  sysdep_routines += memcpy-power8-cached memcpy-power7 memcpy-a2 memcpy-power6 \
 		   strncase-power8
 
 ifneq (,$(filter %le,$(config-machine)))
-sysdep_routines += strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \
+sysdep_routines += memset-power10 \
+		   strcmp-power9 strncmp-power9 strcpy-power9 stpcpy-power9 \
 		   rawmemchr-power9 strlen-power9 strncpy-power9 stpncpy-power9 \
 		   strlen-power10
 endif
diff --git a/sysdeps/powerpc/powerpc64/multiarch/bzero.c b/sysdeps/powerpc/powerpc64/multiarch/bzero.c
index c3f819ff48d6..50a5320c6650 100644
--- a/sysdeps/powerpc/powerpc64/multiarch/bzero.c
+++ b/sysdeps/powerpc/powerpc64/multiarch/bzero.c
@@ -27,8 +27,16 @@  extern __typeof (bzero) __bzero_power4 attribute_hidden;
 extern __typeof (bzero) __bzero_power6 attribute_hidden;
 extern __typeof (bzero) __bzero_power7 attribute_hidden;
 extern __typeof (bzero) __bzero_power8 attribute_hidden;
+# ifdef __LITTLE_ENDIAN__
+extern __typeof (bzero) __bzero_power10 attribute_hidden;
+# endif
 
 libc_ifunc (__bzero,
+# ifdef __LITTLE_ENDIAN__
+	    (hwcap2 & (PPC_FEATURE2_ARCH_3_1 | PPC_FEATURE2_HAS_ISEL)
+	     && hwcap & PPC_FEATURE_HAS_VSX)
+	    ? __bzero_power10 :
+# endif
             (hwcap2 & PPC_FEATURE2_ARCH_2_07)
             ? __bzero_power8 :
 	      (hwcap & PPC_FEATURE_HAS_VSX)
diff --git a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
index 1a6993616f2a..cd0d95ed9a94 100644
--- a/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
+++ b/sysdeps/powerpc/powerpc64/multiarch/ifunc-impl-list.c
@@ -73,6 +73,13 @@  __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   /* Support sysdeps/powerpc/powerpc64/multiarch/memset.c.  */
   IFUNC_IMPL (i, name, memset,
+#ifdef __LITTLE_ENDIAN__
+	      IFUNC_IMPL_ADD (array, i, memset,
+			      hwcap2 & (PPC_FEATURE2_ARCH_3_1 |
+					PPC_FEATURE2_HAS_ISEL)
+			      && hwcap & PPC_FEATURE_HAS_VSX,
+			      __memset_power10)
+#endif
 	      IFUNC_IMPL_ADD (array, i, memset, hwcap2 & PPC_FEATURE2_ARCH_2_07,
 			      __memset_power8)
 	      IFUNC_IMPL_ADD (array, i, memset, hwcap & PPC_FEATURE_HAS_VSX,
@@ -174,6 +181,13 @@  __libc_ifunc_impl_list (const char *name, struct libc_ifunc_impl *array,
 
   /* Support sysdeps/powerpc/powerpc64/multiarch/bzero.c.  */
   IFUNC_IMPL (i, name, bzero,
+#ifdef __LITTLE_ENDIAN__
+	      IFUNC_IMPL_ADD (array, i, bzero,
+			      hwcap2 & (PPC_FEATURE2_ARCH_3_1 |
+					PPC_FEATURE2_HAS_ISEL)
+			      && hwcap & PPC_FEATURE_HAS_VSX,
+			      __bzero_power10)
+#endif
 	      IFUNC_IMPL_ADD (array, i, bzero, hwcap2 & PPC_FEATURE2_ARCH_2_07,
 			      __bzero_power8)
 	      IFUNC_IMPL_ADD (array, i, bzero, hwcap & PPC_FEATURE_HAS_VSX,
diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S
new file mode 100644
index 000000000000..53a9535a2401
--- /dev/null
+++ b/sysdeps/powerpc/powerpc64/multiarch/memset-power10.S
@@ -0,0 +1,27 @@ 
+/* Optimized memset implementation for PowerPC64/POWER10.
+   Copyright (C) 2021 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <https://www.gnu.org/licenses/>.  */
+
+#define MEMSET __memset_power10
+
+#undef libc_hidden_builtin_def
+#define libc_hidden_builtin_def(name)
+
+#undef __bzero
+#define __bzero __bzero_power10
+
+#include <sysdeps/powerpc/powerpc64/le/power10/memset.S>
diff --git a/sysdeps/powerpc/powerpc64/multiarch/memset.c b/sysdeps/powerpc/powerpc64/multiarch/memset.c
index d483f66f2744..6562646dffcf 100644
--- a/sysdeps/powerpc/powerpc64/multiarch/memset.c
+++ b/sysdeps/powerpc/powerpc64/multiarch/memset.c
@@ -33,10 +33,18 @@  extern __typeof (__redirect_memset) __memset_power4 attribute_hidden;
 extern __typeof (__redirect_memset) __memset_power6 attribute_hidden;
 extern __typeof (__redirect_memset) __memset_power7 attribute_hidden;
 extern __typeof (__redirect_memset) __memset_power8 attribute_hidden;
+# ifdef __LITTLE_ENDIAN__
+extern __typeof (__redirect_memset) __memset_power10 attribute_hidden;
+# endif
 
 /* Avoid DWARF definition DIE on ifunc symbol so that GDB can handle
    ifunc symbol properly.  */
 libc_ifunc (__libc_memset,
+# ifdef __LITTLE_ENDIAN__
+	    (hwcap2 & (PPC_FEATURE2_ARCH_3_1 | PPC_FEATURE2_HAS_ISEL)
+	     && hwcap & PPC_FEATURE_HAS_VSX)
+	    ? __memset_power10 :
+# endif
             (hwcap2 & PPC_FEATURE2_ARCH_2_07)
             ? __memset_power8 :
 	      (hwcap & PPC_FEATURE_HAS_VSX)