[01/12] x86: Update vfork to pop shadow stack

Message ID 20180721142035.21059-2-hjl.tools@gmail.com
State New
Headers show
Series
  • x86/CET: The last 12 patches to enable Intel CET
Related show

Commit Message

H.J. Lu July 21, 2018, 2:20 p.m.
Since we can't change return address on shadow stack, if shadow stack
is in use, we need to pop shadow stack and jump back to caller directly.

	* sysdeps/unix/sysv/linux/i386/vfork.S (SYSCALL_ERROR_HANDLER):
	Redefine if shadow stack is enabled.
	(SYSCALL_ERROR_LABEL): Likewise.
	(__vfork): Pop shadow stack and jump back to to caller directly
	when shadow stack is in use.
	* sysdeps/unix/sysv/linux/x86_64/vfork.S (SYSCALL_ERROR_HANDLER):
	Redefine if shadow stack is enabled.
	(SYSCALL_ERROR_LABEL): Likewise.
	(__vfork): Pop shadow stack and jump back to to caller directly
	when shadow stack is in use.
---
 sysdeps/unix/sysv/linux/i386/vfork.S   | 54 ++++++++++++++++++++++++++
 sysdeps/unix/sysv/linux/x86_64/vfork.S | 35 +++++++++++++++++
 2 files changed, 89 insertions(+)

-- 
2.17.1

Comments

Carlos O'Donell July 24, 2018, 2:47 a.m. | #1
On 07/21/2018 10:20 AM, H.J. Lu wrote:
> Since we can't change return address on shadow stack, if shadow stack

> is in use, we need to pop shadow stack and jump back to caller directly.

> 

> 	* sysdeps/unix/sysv/linux/i386/vfork.S (SYSCALL_ERROR_HANDLER):

> 	Redefine if shadow stack is enabled.

> 	(SYSCALL_ERROR_LABEL): Likewise.

> 	(__vfork): Pop shadow stack and jump back to to caller directly

> 	when shadow stack is in use.

> 	* sysdeps/unix/sysv/linux/x86_64/vfork.S (SYSCALL_ERROR_HANDLER):

> 	Redefine if shadow stack is enabled.

> 	(SYSCALL_ERROR_LABEL): Likewise.

> 	(__vfork): Pop shadow stack and jump back to to caller directly

> 	when shadow stack is in use.

> ---

>  sysdeps/unix/sysv/linux/i386/vfork.S   | 54 ++++++++++++++++++++++++++

>  sysdeps/unix/sysv/linux/x86_64/vfork.S | 35 +++++++++++++++++

>  2 files changed, 89 insertions(+)


OK with comment suggestion.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>

 
> diff --git a/sysdeps/unix/sysv/linux/i386/vfork.S b/sysdeps/unix/sysv/linux/i386/vfork.S

> index 8f40d02d09..a75436157e 100644

> --- a/sysdeps/unix/sysv/linux/i386/vfork.S

> +++ b/sysdeps/unix/sysv/linux/i386/vfork.S

> @@ -21,6 +21,35 @@

>  #include <bits/errno.h>

>  #include <tcb-offsets.h>

>  

> +#if SHSTK_ENABLED

> +/* When shadow stack is in use, we need to pop shadow stack and jump

> +   back to caller directly.   */


Suggest:
/* The shadow stack prevents us from pushing the saved return PC
   onto the stack and returning normally.  Instead we pop the shadow
   stack and return directly.  This is the safest way to return
   and ensures any stack manipulations done by the vfork'd child
   doesn't cause the parent to terminate when CET is enabled.  */

> +# undef SYSCALL_ERROR_HANDLER

> +# ifdef PIC

> +#  define SYSCALL_ERROR_HANDLER				\

> +0:							\

> +  calll .L1;						\

> +.L1:							\

> +  popl %edx;						\

> +.L2:							\

> +  addl $_GLOBAL_OFFSET_TABLE_ + (.L2 - .L1), %edx;	\

> +  movl __libc_errno@gotntpoff(%edx), %edx;		\

> +  negl %eax;						\

> +  movl %eax, %gs:(%edx);				\

> +  orl $-1, %eax;					\

> +  jmp 1b;

> +# else

> +#  define SYSCALL_ERROR_HANDLER				\

> +0:							\

> +  movl __libc_errno@indntpoff, %edx;			\

> +  negl %eax;						\

> +  movl %eax, %gs:(%edx);				\

> +  orl $-1, %eax;					\

> +  jmp 1b;

> +# endif

> +# undef SYSCALL_ERROR_LABEL

> +# define SYSCALL_ERROR_LABEL 0f

> +#endif

>  

>  /* Clone the calling process, but without copying the whole address space.

>     The calling process is suspended until the new process exits or is

> @@ -38,16 +67,41 @@ ENTRY (__vfork)

>  	movl	$SYS_ify (vfork), %eax

>  	int	$0x80

>  

> +#if !SHSTK_ENABLED

>  	/* Jump to the return PC.  Don't jump directly since this

>  	   disturbs the branch target cache.  Instead push the return

>  	   address back on the stack.  */

>  	pushl	%ecx

>  	cfi_adjust_cfa_offset (4)

> +#endif


OK.

>  

>  	cmpl	$-4095, %eax

>  	/* Branch forward if it failed.  */

>  	jae	SYSCALL_ERROR_LABEL

>  

> +#if SHSTK_ENABLED

> +1:

> +	/* Check if shadow stack is in use.  */

> +	xorl	%edx, %edx

> +	rdsspd	%edx

> +	testl	%edx, %edx

> +	/* Normal return if shadow stack isn't in use.  */

> +	je	L(no_shstk)


OK.

> +

> +	/* Pop return address from shadow stack and jump back to caller

> +	   directly.  */

> +	movl	$1, %edx

> +	incsspd	%edx

> +	jmp	*%ecx


OK.

> +

> +L(no_shstk):

> +	/* Jump to the return PC.  Don't jump directly since this

> +	   disturbs the branch target cache.  Instead push the return

> +	   address back on the stack.  */

> +	pushl	%ecx

> +	cfi_adjust_cfa_offset (4)

> +#endif

> +

>  	ret

>  

>  PSEUDO_END (__vfork)

> diff --git a/sysdeps/unix/sysv/linux/x86_64/vfork.S b/sysdeps/unix/sysv/linux/x86_64/vfork.S

> index e4c8269e3d..a6d6280bd2 100644

> --- a/sysdeps/unix/sysv/linux/x86_64/vfork.S

> +++ b/sysdeps/unix/sysv/linux/x86_64/vfork.S

> @@ -20,6 +20,18 @@

>  #include <bits/errno.h>

>  #include <tcb-offsets.h>

>  

> +#if SHSTK_ENABLED

> +/* When shadow stack is in use, we need to pop shadow stack and jump

> +   back to caller directly.   */


Likewise.

> +# undef SYSCALL_ERROR_HANDLER

> +# define SYSCALL_ERROR_HANDLER			\

> +0:						\

> +  SYSCALL_SET_ERRNO;				\

> +  or $-1, %RAX_LP;				\

> +  jmp 1b;

> +# undef SYSCALL_ERROR_LABEL

> +# define SYSCALL_ERROR_LABEL 0f

> +#endif

>  

>  /* Clone the calling process, but without copying the whole address space.

>     The calling process is suspended until the new process exits or is

> @@ -38,13 +50,36 @@ ENTRY (__vfork)

>  	movl	$SYS_ify (vfork), %eax

>  	syscall

>  

> +#if !SHSTK_ENABLED

>  	/* Push back the return PC.  */

>  	pushq	%rdi

>  	cfi_adjust_cfa_offset(8)

> +#endif

>  

>  	cmpl	$-4095, %eax

>  	jae SYSCALL_ERROR_LABEL		/* Branch forward if it failed.  */

>  

> +#if SHSTK_ENABLED

> +1:

> +	/* Check if shadow stack is in use.  */

> +	xorl	%esi, %esi

> +	rdsspq	%rsi

> +	testq	%rsi, %rsi

> +	/* Normal return if shadow stack isn't in use.  */

> +	je	L(no_shstk)


OK.

> +

> +	/* Pop return address from shadow stack and jump back to caller

> +	   directly.  */

> +	movl	$1, %esi

> +	incsspq	%rsi

> +	jmp	*%rdi

> +

> +L(no_shstk):

> +	/* Push back the return PC.  */

> +	pushq	%rdi

> +	cfi_adjust_cfa_offset(8)

> +#endif

> +


OK.

>  	/* Normal return.  */

>  	ret

>  

>
H.J. Lu July 24, 2018, 12:25 p.m. | #2
On Mon, Jul 23, 2018 at 7:47 PM, Carlos O'Donell <carlos@redhat.com> wrote:
> On 07/21/2018 10:20 AM, H.J. Lu wrote:

>> Since we can't change return address on shadow stack, if shadow stack

>> is in use, we need to pop shadow stack and jump back to caller directly.

>>

>>       * sysdeps/unix/sysv/linux/i386/vfork.S (SYSCALL_ERROR_HANDLER):

>>       Redefine if shadow stack is enabled.

>>       (SYSCALL_ERROR_LABEL): Likewise.

>>       (__vfork): Pop shadow stack and jump back to to caller directly

>>       when shadow stack is in use.

>>       * sysdeps/unix/sysv/linux/x86_64/vfork.S (SYSCALL_ERROR_HANDLER):

>>       Redefine if shadow stack is enabled.

>>       (SYSCALL_ERROR_LABEL): Likewise.

>>       (__vfork): Pop shadow stack and jump back to to caller directly

>>       when shadow stack is in use.

>> ---

>>  sysdeps/unix/sysv/linux/i386/vfork.S   | 54 ++++++++++++++++++++++++++

>>  sysdeps/unix/sysv/linux/x86_64/vfork.S | 35 +++++++++++++++++

>>  2 files changed, 89 insertions(+)

>

> OK with comment suggestion.

>

> Reviewed-by: Carlos O'Donell <carlos@redhat.com>

>


This is what I checked in.

Thanks.

-- 
H.J.
From 2f165ed718036e91a798a657dc00101393692208 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Thu, 7 Jun 2018 20:50:11 -0700
Subject: [PATCH] Add <bits/indirect-return.h>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Add <bits/indirect-return.h> and include it in <ucontext.h>.
__INDIRECT_RETURN defined in <bits/indirect-return.h> indicates if
swapcontext requires special compiler treatment.  The default
__INDIRECT_RETURN is empty.

On x86, when shadow stack is enabled, __INDIRECT_RETURN is defined
with indirect_return attribute, which has been added to GCC 9, to
indicate that swapcontext returns via indirect branch.  Otherwise
__INDIRECT_RETURN is defined with returns_twice attribute.

When shadow stack is enabled, remove always_inline attribute from
prepare_test_buffer in string/tst-xbzero-opt.c to avoid:

tst-xbzero-opt.c: In function ‘prepare_test_buffer’:
tst-xbzero-opt.c:105:1: error: function ‘prepare_test_buffer’ can never be inlined because it uses setjmp
 prepare_test_buffer (unsigned char *buf)

when indirect_return attribute isn't available.

	* bits/indirect-return.h: New file.
	* misc/sys/cdefs.h (__glibc_has_attribute): New.
	* sysdeps/x86/bits/indirect-return.h: Likewise.
	* stdlib/Makefile (headers): Add bits/indirect-return.h.
	* stdlib/ucontext.h: Include <bits/indirect-return.h>.
	(swapcontext): Add __INDIRECT_RETURN.
	* string/tst-xbzero-opt.c (ALWAYS_INLINE): New.
	(prepare_test_buffer): Use it.
---
 bits/indirect-return.h             | 25 +++++++++++++++++++++
 misc/sys/cdefs.h                   |  6 +++++
 stdlib/Makefile                    |  2 +-
 stdlib/ucontext.h                  |  6 ++++-
 string/tst-xbzero-opt.c            | 10 ++++++++-
 sysdeps/x86/bits/indirect-return.h | 35 ++++++++++++++++++++++++++++++
 6 files changed, 81 insertions(+), 3 deletions(-)
 create mode 100644 bits/indirect-return.h
 create mode 100644 sysdeps/x86/bits/indirect-return.h

diff --git a/bits/indirect-return.h b/bits/indirect-return.h
new file mode 100644
index 0000000000..47f6f15a6e
--- /dev/null
+++ b/bits/indirect-return.h
@@ -0,0 +1,25 @@
+/* Definition of __INDIRECT_RETURN.  Generic version.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef _UCONTEXT_H
+# error "Never include <bits/indirect-return.h> directly; use <ucontext.h> instead."
+#endif
+
+/* __INDIRECT_RETURN is used on swapcontext to indicate if it requires
+   special compiler treatment.  */
+#define __INDIRECT_RETURN
diff --git a/misc/sys/cdefs.h b/misc/sys/cdefs.h
index e80a45ca68..3f6fe3cc85 100644
--- a/misc/sys/cdefs.h
+++ b/misc/sys/cdefs.h
@@ -406,6 +406,12 @@
 # define __glibc_likely(cond)	(cond)
 #endif
 
+#ifdef __has_attribute
+# define __glibc_has_attribute(attr)	__has_attribute (attr)
+#else
+# define __glibc_has_attribute(attr)	0
+#endif
+
 #if (!defined _Noreturn \
      && (defined __STDC_VERSION__ ? __STDC_VERSION__ : 0) < 201112 \
      &&  !__GNUC_PREREQ (4,7))
diff --git a/stdlib/Makefile b/stdlib/Makefile
index 808a8ceab7..b5e55b0a55 100644
--- a/stdlib/Makefile
+++ b/stdlib/Makefile
@@ -26,7 +26,7 @@ headers	:= stdlib.h bits/stdlib.h bits/stdlib-ldbl.h bits/stdlib-float.h      \
 	   monetary.h bits/monetary-ldbl.h				      \
 	   inttypes.h stdint.h bits/wordsize.h				      \
 	   errno.h sys/errno.h bits/errno.h bits/types/error_t.h	      \
-	   ucontext.h sys/ucontext.h					      \
+	   ucontext.h sys/ucontext.h bits/indirect-return.h		      \
 	   alloca.h fmtmsg.h						      \
 	   bits/stdlib-bsearch.h sys/random.h bits/stdint-intn.h	      \
 	   bits/stdint-uintn.h
diff --git a/stdlib/ucontext.h b/stdlib/ucontext.h
index eec7611631..ec630038f6 100644
--- a/stdlib/ucontext.h
+++ b/stdlib/ucontext.h
@@ -22,6 +22,9 @@
 
 #include <features.h>
 
+/* Get definition of __INDIRECT_RETURN.  */
+#include <bits/indirect-return.h>
+
 /* Get machine dependent definition of data structures.  */
 #include <sys/ucontext.h>
 
@@ -36,7 +39,8 @@ extern int setcontext (const ucontext_t *__ucp) __THROWNL;
 /* Save current context in context variable pointed to by OUCP and set
    context from variable pointed to by UCP.  */
 extern int swapcontext (ucontext_t *__restrict __oucp,
-			const ucontext_t *__restrict __ucp) __THROWNL;
+			const ucontext_t *__restrict __ucp)
+  __THROWNL __INDIRECT_RETURN;
 
 /* Manipulate user context UCP to continue with calling functions FUNC
    and the ARGC-1 parameters following ARGC when the context is used
diff --git a/string/tst-xbzero-opt.c b/string/tst-xbzero-opt.c
index cf7041f37a..aab4a7f715 100644
--- a/string/tst-xbzero-opt.c
+++ b/string/tst-xbzero-opt.c
@@ -100,7 +100,15 @@ static ucontext_t uc_main, uc_co;
 /* Always check the test buffer immediately after filling it; this
    makes externally visible side effects depend on the buffer existing
    and having been filled in.  */
-static inline __attribute__  ((always_inline)) void
+#if defined __CET__ && !__glibc_has_attribute (__indirect_return__)
+/* Note: swapcontext returns via indirect branch when SHSTK is enabled.
+   Without indirect_return attribute, swapcontext is marked with
+   returns_twice attribute, which prevents always_inline to work.  */
+# define ALWAYS_INLINE
+#else
+# define ALWAYS_INLINE	__attribute__ ((always_inline))
+#endif
+static inline ALWAYS_INLINE void
 prepare_test_buffer (unsigned char *buf)
 {
   for (unsigned int i = 0; i < PATTERN_REPS; i++)
diff --git a/sysdeps/x86/bits/indirect-return.h b/sysdeps/x86/bits/indirect-return.h
new file mode 100644
index 0000000000..0587e687ac
--- /dev/null
+++ b/sysdeps/x86/bits/indirect-return.h
@@ -0,0 +1,35 @@
+/* Definition of __INDIRECT_RETURN.  x86 version.
+   Copyright (C) 2018 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef _UCONTEXT_H
+# error "Never include <bits/indirect-return.h> directly; use <ucontext.h> instead."
+#endif
+
+/* On x86, swapcontext returns via indirect branch when the shadow stack
+   is enabled.  Define __INDIRECT_RETURN to indicate whether swapcontext
+   returns via indirect branch.  */
+#if defined __CET__ && (__CET__ & 2) != 0
+# if __glibc_has_attribute (__indirect_return__)
+#  define __INDIRECT_RETURN __attribute__ ((__indirect_return__))
+# else
+/* Without indirect_return attribute, use returns_twice attribute.  */
+#  define __INDIRECT_RETURN __attribute__ ((__returns_twice__))
+# endif
+#else
+# define __INDIRECT_RETURN
+#endif
Carlos O'Donell July 24, 2018, 12:32 p.m. | #3
On 07/24/2018 08:25 AM, H.J. Lu wrote:
> On Mon, Jul 23, 2018 at 7:47 PM, Carlos O'Donell <carlos@redhat.com> wrote:

>> On 07/21/2018 10:20 AM, H.J. Lu wrote:

>>> Since we can't change return address on shadow stack, if shadow stack

>>> is in use, we need to pop shadow stack and jump back to caller directly.

>>>

>>>       * sysdeps/unix/sysv/linux/i386/vfork.S (SYSCALL_ERROR_HANDLER):

>>>       Redefine if shadow stack is enabled.

>>>       (SYSCALL_ERROR_LABEL): Likewise.

>>>       (__vfork): Pop shadow stack and jump back to to caller directly

>>>       when shadow stack is in use.

>>>       * sysdeps/unix/sysv/linux/x86_64/vfork.S (SYSCALL_ERROR_HANDLER):

>>>       Redefine if shadow stack is enabled.

>>>       (SYSCALL_ERROR_LABEL): Likewise.

>>>       (__vfork): Pop shadow stack and jump back to to caller directly

>>>       when shadow stack is in use.

>>> ---

>>>  sysdeps/unix/sysv/linux/i386/vfork.S   | 54 ++++++++++++++++++++++++++

>>>  sysdeps/unix/sysv/linux/x86_64/vfork.S | 35 +++++++++++++++++

>>>  2 files changed, 89 insertions(+)

>>

>> OK with comment suggestion.

>>

>> Reviewed-by: Carlos O'Donell <carlos@redhat.com>

>>

> 

> This is what I checked in.

> 

> Thanks.

> 


Wrong patch?

c.
H.J. Lu July 24, 2018, 1:17 p.m. | #4
On Tue, Jul 24, 2018 at 5:32 AM, Carlos O'Donell <carlos@redhat.com> wrote:
> On 07/24/2018 08:25 AM, H.J. Lu wrote:

>> On Mon, Jul 23, 2018 at 7:47 PM, Carlos O'Donell <carlos@redhat.com> wrote:

>>> On 07/21/2018 10:20 AM, H.J. Lu wrote:

>>>> Since we can't change return address on shadow stack, if shadow stack

>>>> is in use, we need to pop shadow stack and jump back to caller directly.

>>>>

>>>>       * sysdeps/unix/sysv/linux/i386/vfork.S (SYSCALL_ERROR_HANDLER):

>>>>       Redefine if shadow stack is enabled.

>>>>       (SYSCALL_ERROR_LABEL): Likewise.

>>>>       (__vfork): Pop shadow stack and jump back to to caller directly

>>>>       when shadow stack is in use.

>>>>       * sysdeps/unix/sysv/linux/x86_64/vfork.S (SYSCALL_ERROR_HANDLER):

>>>>       Redefine if shadow stack is enabled.

>>>>       (SYSCALL_ERROR_LABEL): Likewise.

>>>>       (__vfork): Pop shadow stack and jump back to to caller directly

>>>>       when shadow stack is in use.

>>>> ---

>>>>  sysdeps/unix/sysv/linux/i386/vfork.S   | 54 ++++++++++++++++++++++++++

>>>>  sysdeps/unix/sysv/linux/x86_64/vfork.S | 35 +++++++++++++++++

>>>>  2 files changed, 89 insertions(+)

>>>

>>> OK with comment suggestion.

>>>

>>> Reviewed-by: Carlos O'Donell <carlos@redhat.com>

>>>

>>

>> This is what I checked in.

>>

>> Thanks.

>>

>

> Wrong patch?

>


Oops.  Here is the right one.

-- 
H.J.
From 3650e1d9284926be67e368707a9a2276fb77a167 Mon Sep 17 00:00:00 2001
From: "H.J. Lu" <hjl.tools@gmail.com>
Date: Tue, 24 Jul 2018 05:20:48 -0700
Subject: [PATCH] x86: Update vfork to pop shadow stack

The shadow stack prevents us from pushing the saved return PC onto
the stack and returning normally.  Instead we pop the shadow stack
and return directly.  This is the safest way to return and ensures
any stack manipulations done by the vfork'd child doesn't cause the
parent to terminate when CET is enabled.

Reviewed-by: Carlos O'Donell <carlos@redhat.com>

	* sysdeps/unix/sysv/linux/i386/vfork.S (SYSCALL_ERROR_HANDLER):
	Redefine if shadow stack is enabled.
	(SYSCALL_ERROR_LABEL): Likewise.
	(__vfork): Pop shadow stack and jump back to to caller directly
	when shadow stack is in use.
	* sysdeps/unix/sysv/linux/x86_64/vfork.S (SYSCALL_ERROR_HANDLER):
	Redefine if shadow stack is enabled.
	(SYSCALL_ERROR_LABEL): Likewise.
	(__vfork): Pop shadow stack and jump back to to caller directly
	when shadow stack is in use.
---
 ChangeLog                              | 13 ++++++
 sysdeps/unix/sysv/linux/i386/vfork.S   | 57 ++++++++++++++++++++++++++
 sysdeps/unix/sysv/linux/x86_64/vfork.S | 38 +++++++++++++++++
 3 files changed, 108 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2a1a53d9df..8b892eb686 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+2018-07-24  H.J. Lu  <hongjiu.lu@intel.com>
+
+	* sysdeps/unix/sysv/linux/i386/vfork.S (SYSCALL_ERROR_HANDLER):
+	Redefine if shadow stack is enabled.
+	(SYSCALL_ERROR_LABEL): Likewise.
+	(__vfork): Pop shadow stack and jump back to to caller directly
+	when shadow stack is in use.
+	* sysdeps/unix/sysv/linux/x86_64/vfork.S (SYSCALL_ERROR_HANDLER):
+	Redefine if shadow stack is enabled.
+	(SYSCALL_ERROR_LABEL): Likewise.
+	(__vfork): Pop shadow stack and jump back to to caller directly
+	when shadow stack is in use.
+
 2018-07-24  H.J. Lu  <hongjiu.lu@intel.com>
 
 	* sysdeps/x86_64/tst-quadmod1.S (func): Add endbr64 if IBT is
diff --git a/sysdeps/unix/sysv/linux/i386/vfork.S b/sysdeps/unix/sysv/linux/i386/vfork.S
index 8f40d02d09..ce6dbfac48 100644
--- a/sysdeps/unix/sysv/linux/i386/vfork.S
+++ b/sysdeps/unix/sysv/linux/i386/vfork.S
@@ -21,6 +21,38 @@
 #include <bits/errno.h>
 #include <tcb-offsets.h>
 
+#if SHSTK_ENABLED
+/* The shadow stack prevents us from pushing the saved return PC onto
+   the stack and returning normally.  Instead we pop the shadow stack
+   and return directly.  This is the safest way to return and ensures
+   any stack manipulations done by the vfork'd child doesn't cause the
+   parent to terminate when CET is enabled.  */
+# undef SYSCALL_ERROR_HANDLER
+# ifdef PIC
+#  define SYSCALL_ERROR_HANDLER				\
+0:							\
+  calll .L1;						\
+.L1:							\
+  popl %edx;						\
+.L2:							\
+  addl $_GLOBAL_OFFSET_TABLE_ + (.L2 - .L1), %edx;	\
+  movl __libc_errno@gotntpoff(%edx), %edx;		\
+  negl %eax;						\
+  movl %eax, %gs:(%edx);				\
+  orl $-1, %eax;					\
+  jmp 1b;
+# else
+#  define SYSCALL_ERROR_HANDLER				\
+0:							\
+  movl __libc_errno@indntpoff, %edx;			\
+  negl %eax;						\
+  movl %eax, %gs:(%edx);				\
+  orl $-1, %eax;					\
+  jmp 1b;
+# endif
+# undef SYSCALL_ERROR_LABEL
+# define SYSCALL_ERROR_LABEL 0f
+#endif
 
 /* Clone the calling process, but without copying the whole address space.
    The calling process is suspended until the new process exits or is
@@ -38,16 +70,41 @@ ENTRY (__vfork)
 	movl	$SYS_ify (vfork), %eax
 	int	$0x80
 
+#if !SHSTK_ENABLED
 	/* Jump to the return PC.  Don't jump directly since this
 	   disturbs the branch target cache.  Instead push the return
 	   address back on the stack.  */
 	pushl	%ecx
 	cfi_adjust_cfa_offset (4)
+#endif
 
 	cmpl	$-4095, %eax
 	/* Branch forward if it failed.  */
 	jae	SYSCALL_ERROR_LABEL
 
+#if SHSTK_ENABLED
+1:
+	/* Check if shadow stack is in use.  */
+	xorl	%edx, %edx
+	rdsspd	%edx
+	testl	%edx, %edx
+	/* Normal return if shadow stack isn't in use.  */
+	je	L(no_shstk)
+
+	/* Pop return address from shadow stack and jump back to caller
+	   directly.  */
+	movl	$1, %edx
+	incsspd	%edx
+	jmp	*%ecx
+
+L(no_shstk):
+	/* Jump to the return PC.  Don't jump directly since this
+	   disturbs the branch target cache.  Instead push the return
+	   address back on the stack.  */
+	pushl	%ecx
+	cfi_adjust_cfa_offset (4)
+#endif
+
 	ret
 
 PSEUDO_END (__vfork)
diff --git a/sysdeps/unix/sysv/linux/x86_64/vfork.S b/sysdeps/unix/sysv/linux/x86_64/vfork.S
index e4c8269e3d..8f1ca9f836 100644
--- a/sysdeps/unix/sysv/linux/x86_64/vfork.S
+++ b/sysdeps/unix/sysv/linux/x86_64/vfork.S
@@ -20,6 +20,21 @@
 #include <bits/errno.h>
 #include <tcb-offsets.h>
 
+#if SHSTK_ENABLED
+/* The shadow stack prevents us from pushing the saved return PC onto
+   the stack and returning normally.  Instead we pop the shadow stack
+   and return directly.  This is the safest way to return and ensures
+   any stack manipulations done by the vfork'd child doesn't cause the
+   parent to terminate when CET is enabled.  */
+# undef SYSCALL_ERROR_HANDLER
+# define SYSCALL_ERROR_HANDLER			\
+0:						\
+  SYSCALL_SET_ERRNO;				\
+  or $-1, %RAX_LP;				\
+  jmp 1b;
+# undef SYSCALL_ERROR_LABEL
+# define SYSCALL_ERROR_LABEL 0f
+#endif
 
 /* Clone the calling process, but without copying the whole address space.
    The calling process is suspended until the new process exits or is
@@ -38,13 +53,36 @@ ENTRY (__vfork)
 	movl	$SYS_ify (vfork), %eax
 	syscall
 
+#if !SHSTK_ENABLED
 	/* Push back the return PC.  */
 	pushq	%rdi
 	cfi_adjust_cfa_offset(8)
+#endif
 
 	cmpl	$-4095, %eax
 	jae SYSCALL_ERROR_LABEL		/* Branch forward if it failed.  */
 
+#if SHSTK_ENABLED
+1:
+	/* Check if shadow stack is in use.  */
+	xorl	%esi, %esi
+	rdsspq	%rsi
+	testq	%rsi, %rsi
+	/* Normal return if shadow stack isn't in use.  */
+	je	L(no_shstk)
+
+	/* Pop return address from shadow stack and jump back to caller
+	   directly.  */
+	movl	$1, %esi
+	incsspq	%rsi
+	jmp	*%rdi
+
+L(no_shstk):
+	/* Push back the return PC.  */
+	pushq	%rdi
+	cfi_adjust_cfa_offset(8)
+#endif
+
 	/* Normal return.  */
 	ret

Patch

diff --git a/sysdeps/unix/sysv/linux/i386/vfork.S b/sysdeps/unix/sysv/linux/i386/vfork.S
index 8f40d02d09..a75436157e 100644
--- a/sysdeps/unix/sysv/linux/i386/vfork.S
+++ b/sysdeps/unix/sysv/linux/i386/vfork.S
@@ -21,6 +21,35 @@ 
 #include <bits/errno.h>
 #include <tcb-offsets.h>
 
+#if SHSTK_ENABLED
+/* When shadow stack is in use, we need to pop shadow stack and jump
+   back to caller directly.   */
+# undef SYSCALL_ERROR_HANDLER
+# ifdef PIC
+#  define SYSCALL_ERROR_HANDLER				\
+0:							\
+  calll .L1;						\
+.L1:							\
+  popl %edx;						\
+.L2:							\
+  addl $_GLOBAL_OFFSET_TABLE_ + (.L2 - .L1), %edx;	\
+  movl __libc_errno@gotntpoff(%edx), %edx;		\
+  negl %eax;						\
+  movl %eax, %gs:(%edx);				\
+  orl $-1, %eax;					\
+  jmp 1b;
+# else
+#  define SYSCALL_ERROR_HANDLER				\
+0:							\
+  movl __libc_errno@indntpoff, %edx;			\
+  negl %eax;						\
+  movl %eax, %gs:(%edx);				\
+  orl $-1, %eax;					\
+  jmp 1b;
+# endif
+# undef SYSCALL_ERROR_LABEL
+# define SYSCALL_ERROR_LABEL 0f
+#endif
 
 /* Clone the calling process, but without copying the whole address space.
    The calling process is suspended until the new process exits or is
@@ -38,16 +67,41 @@  ENTRY (__vfork)
 	movl	$SYS_ify (vfork), %eax
 	int	$0x80
 
+#if !SHSTK_ENABLED
 	/* Jump to the return PC.  Don't jump directly since this
 	   disturbs the branch target cache.  Instead push the return
 	   address back on the stack.  */
 	pushl	%ecx
 	cfi_adjust_cfa_offset (4)
+#endif
 
 	cmpl	$-4095, %eax
 	/* Branch forward if it failed.  */
 	jae	SYSCALL_ERROR_LABEL
 
+#if SHSTK_ENABLED
+1:
+	/* Check if shadow stack is in use.  */
+	xorl	%edx, %edx
+	rdsspd	%edx
+	testl	%edx, %edx
+	/* Normal return if shadow stack isn't in use.  */
+	je	L(no_shstk)
+
+	/* Pop return address from shadow stack and jump back to caller
+	   directly.  */
+	movl	$1, %edx
+	incsspd	%edx
+	jmp	*%ecx
+
+L(no_shstk):
+	/* Jump to the return PC.  Don't jump directly since this
+	   disturbs the branch target cache.  Instead push the return
+	   address back on the stack.  */
+	pushl	%ecx
+	cfi_adjust_cfa_offset (4)
+#endif
+
 	ret
 
 PSEUDO_END (__vfork)
diff --git a/sysdeps/unix/sysv/linux/x86_64/vfork.S b/sysdeps/unix/sysv/linux/x86_64/vfork.S
index e4c8269e3d..a6d6280bd2 100644
--- a/sysdeps/unix/sysv/linux/x86_64/vfork.S
+++ b/sysdeps/unix/sysv/linux/x86_64/vfork.S
@@ -20,6 +20,18 @@ 
 #include <bits/errno.h>
 #include <tcb-offsets.h>
 
+#if SHSTK_ENABLED
+/* When shadow stack is in use, we need to pop shadow stack and jump
+   back to caller directly.   */
+# undef SYSCALL_ERROR_HANDLER
+# define SYSCALL_ERROR_HANDLER			\
+0:						\
+  SYSCALL_SET_ERRNO;				\
+  or $-1, %RAX_LP;				\
+  jmp 1b;
+# undef SYSCALL_ERROR_LABEL
+# define SYSCALL_ERROR_LABEL 0f
+#endif
 
 /* Clone the calling process, but without copying the whole address space.
    The calling process is suspended until the new process exits or is
@@ -38,13 +50,36 @@  ENTRY (__vfork)
 	movl	$SYS_ify (vfork), %eax
 	syscall
 
+#if !SHSTK_ENABLED
 	/* Push back the return PC.  */
 	pushq	%rdi
 	cfi_adjust_cfa_offset(8)
+#endif
 
 	cmpl	$-4095, %eax
 	jae SYSCALL_ERROR_LABEL		/* Branch forward if it failed.  */
 
+#if SHSTK_ENABLED
+1:
+	/* Check if shadow stack is in use.  */
+	xorl	%esi, %esi
+	rdsspq	%rsi
+	testq	%rsi, %rsi
+	/* Normal return if shadow stack isn't in use.  */
+	je	L(no_shstk)
+
+	/* Pop return address from shadow stack and jump back to caller
+	   directly.  */
+	movl	$1, %esi
+	incsspq	%rsi
+	jmp	*%rdi
+
+L(no_shstk):
+	/* Push back the return PC.  */
+	pushq	%rdi
+	cfi_adjust_cfa_offset(8)
+#endif
+
 	/* Normal return.  */
 	ret