elf, nptl: Initialize static TLS directly in ld.so

Message ID 87czu6ihok.fsf@oldenburg.str.redhat.com
State New
Headers show
Series
  • elf, nptl: Initialize static TLS directly in ld.so
Related show

Commit Message

Stefan Liebler via Libc-alpha May 4, 2021, 4:31 p.m.
The stack list is available in ld.so since commit
1daccf403b1bd86370eb94edca794dc106d02039 ("nptl: Move stack list
variables into _rtld_global"), so it's possible to walk the stack
list directly in ld.so and perform the initialization there.

This eliminates an unprotected function pointer from _rtld_global
and reduces the libpthread initialization code.

---
 elf/dl-open.c              |  2 +-
 elf/dl-reloc.c             |  5 +++--
 elf/dl-support.c           |  3 +--
 elf/dl-tls.c               | 39 +++++++++++++++++++++++++++++++++++++++
 elf/rtld.c                 |  2 ++
 nptl/allocatestack.c       | 35 -----------------------------------
 nptl/nptl-init.c           |  2 --
 nptl/pthreadP.h            |  2 --
 sysdeps/generic/ldsodefs.h | 19 +++++++++++++++++++
 9 files changed, 65 insertions(+), 44 deletions(-)

Comments

Stefan Liebler via Libc-alpha May 4, 2021, 8:42 p.m. | #1
On Tue, May 4, 2021 at 12:01 PM Florian Weimer via Libc-alpha
<libc-alpha@sourceware.org> wrote:
>

> The stack list is available in ld.so since commit

> 1daccf403b1bd86370eb94edca794dc106d02039 ("nptl: Move stack list

> variables into _rtld_global"), so it's possible to walk the stack

> list directly in ld.so and perform the initialization there.

>

> This eliminates an unprotected function pointer from _rtld_global

> and reduces the libpthread initialization code.

>

> ---

>  elf/dl-open.c              |  2 +-

>  elf/dl-reloc.c             |  5 +++--

>  elf/dl-support.c           |  3 +--

>  elf/dl-tls.c               | 39 +++++++++++++++++++++++++++++++++++++++

>  elf/rtld.c                 |  2 ++

>  nptl/allocatestack.c       | 35 -----------------------------------

>  nptl/nptl-init.c           |  2 --

>  nptl/pthreadP.h            |  2 --

>  sysdeps/generic/ldsodefs.h | 19 +++++++++++++++++++

>  9 files changed, 65 insertions(+), 44 deletions(-)

>

> diff --git a/elf/dl-open.c b/elf/dl-open.c

> index ab7aaa345e..09f0df7d38 100644

> --- a/elf/dl-open.c

> +++ b/elf/dl-open.c

> @@ -426,7 +426,7 @@ TLS generation counter wrapped!  Please report this."));

>           _dl_update_slotinfo (imap->l_tls_modid);

>  #endif

>

> -         GL(dl_init_static_tls) (imap);

> +         dl_init_static_tls (imap);

>           assert (imap->l_need_tls_init == 0);

>         }

>      }

> diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c

> index c2df26deea..bb9ca1a101 100644

> --- a/elf/dl-reloc.c

> +++ b/elf/dl-reloc.c

> @@ -118,7 +118,7 @@ _dl_try_allocate_static_tls (struct link_map *map, bool optional)

>         (void) _dl_update_slotinfo (map->l_tls_modid);

>  #endif

>

> -      GL(dl_init_static_tls) (map);

> +      dl_init_static_tls (map);

>      }

>    else

>      map->l_need_tls_init = 1;

> @@ -141,6 +141,7 @@ cannot allocate memory in static TLS block"));

>      }

>  }

>

> +#if !THREAD_GSCOPE_IN_TCB

>  /* Initialize static TLS area and DTV for current (only) thread.

>     libpthread implementations should provide their own hook

>     to handle all threads.  */

> @@ -159,7 +160,7 @@ _dl_nothread_init_static_tls (struct link_map *map)

>    memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),

>           '\0', map->l_tls_blocksize - map->l_tls_initimage_size);

>  }

> -

> +#endif /* !THREAD_GSCOPE_IN_TCB */

>

>  void

>  _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[],

> diff --git a/elf/dl-support.c b/elf/dl-support.c

> index 7fc2ee78e2..f966a2e7cd 100644

> --- a/elf/dl-support.c

> +++ b/elf/dl-support.c

> @@ -138,8 +138,6 @@ void *_dl_random;

>  #include <dl-procruntime.c>

>  #include <dl-procinfo.c>

>

> -void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;

> -

>  size_t _dl_pagesize = EXEC_PAGESIZE;

>

>  size_t _dl_minsigstacksize = CONSTANT_MINSIGSTKSZ;

> @@ -197,6 +195,7 @@ list_t _dl_stack_user;

>  int _dl_stack_cache_lock;

>  #else

>  int _dl_thread_gscope_count;

> +void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;

>  #endif

>  struct dl_scope_free_list *_dl_scope_free_list;

>

> diff --git a/elf/dl-tls.c b/elf/dl-tls.c

> index f8b32b3ecb..6baff0c1ea 100644

> --- a/elf/dl-tls.c

> +++ b/elf/dl-tls.c

> @@ -29,6 +29,10 @@

>  #include <dl-tls.h>

>  #include <ldsodefs.h>

>

> +#if THREAD_GSCOPE_IN_TCB

> +# include <list.h>

> +#endif

> +

>  #define TUNABLE_NAMESPACE rtld

>  #include <dl-tunables.h>

>

> @@ -1005,3 +1009,38 @@ cannot create TLS data structures"));

>        listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;

>      }

>  }

> +

> +#if THREAD_GSCOPE_IN_TCB

> +static inline void __attribute__((always_inline))

> +init_one_static_tls (struct pthread *curp, struct link_map *map)

> +{

> +# if TLS_TCB_AT_TP

> +  void *dest = (char *) curp - map->l_tls_offset;

> +# elif TLS_DTV_AT_TP

> +  void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;

> +# else

> +#  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"

> +# endif

> +

> +  /* Initialize the memory.  */

> +  memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),

> +         '\0', map->l_tls_blocksize - map->l_tls_initimage_size);

> +}

> +

> +void

> +_dl_init_static_tls (struct link_map *map)

> +{

> +  lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);

> +

> +  /* Iterate over the list with system-allocated threads first.  */

> +  list_t *runp;

> +  list_for_each (runp, &GL (dl_stack_used))

> +    init_one_static_tls (list_entry (runp, struct pthread, list), map);

> +

> +  /* Now the list with threads using user-allocated stacks.  */

> +  list_for_each (runp, &GL (dl_stack_user))

> +    init_one_static_tls (list_entry (runp, struct pthread, list), map);

> +

> +  lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);

> +}

> +#endif /* THREAD_GSCOPE_IN_TCB */

> diff --git a/elf/rtld.c b/elf/rtld.c

> index 34879016ad..ad325d4c10 100644

> --- a/elf/rtld.c

> +++ b/elf/rtld.c

> @@ -1139,7 +1139,9 @@ dl_main (const ElfW(Phdr) *phdr,

>    struct dl_main_state state;

>    dl_main_state_init (&state);

>

> +#if !THREAD_GSCOPE_IN_TCB

>    GL(dl_init_static_tls) = &_dl_nothread_init_static_tls;

> +#endif

>

>  #if defined SHARED && defined _LIBC_REENTRANT \

>      && defined __rtld_lock_default_lock_recursive

> diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c

> index f1270c3109..8aaba088b1 100644

> --- a/nptl/allocatestack.c

> +++ b/nptl/allocatestack.c

> @@ -951,38 +951,3 @@ __reclaim_stacks (void)

>    GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER;

>    __default_pthread_attr_lock = LLL_LOCK_INITIALIZER;

>  }

> -

> -

> -static inline void __attribute__((always_inline))

> -init_one_static_tls (struct pthread *curp, struct link_map *map)

> -{

> -# if TLS_TCB_AT_TP

> -  void *dest = (char *) curp - map->l_tls_offset;

> -# elif TLS_DTV_AT_TP

> -  void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;

> -# else

> -#  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"

> -# endif

> -

> -  /* Initialize the memory.  */

> -  memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),

> -         '\0', map->l_tls_blocksize - map->l_tls_initimage_size);

> -}

> -

> -void

> -attribute_hidden

> -__pthread_init_static_tls (struct link_map *map)

> -{

> -  lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);

> -

> -  /* Iterate over the list with system-allocated threads first.  */

> -  list_t *runp;

> -  list_for_each (runp, &GL (dl_stack_used))

> -    init_one_static_tls (list_entry (runp, struct pthread, list), map);

> -

> -  /* Now the list with threads using user-allocated stacks.  */

> -  list_for_each (runp, &GL (dl_stack_user))

> -    init_one_static_tls (list_entry (runp, struct pthread, list), map);

> -

> -  lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);

> -}

> diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c

> index b0879bd87e..fcab5a0904 100644

> --- a/nptl/nptl-init.c

> +++ b/nptl/nptl-init.c

> @@ -191,8 +191,6 @@ __pthread_initialize_minimal_internal (void)

>    GL(dl_make_stack_executable_hook) = &__make_stacks_executable;

>  #endif

>

> -  GL(dl_init_static_tls) = &__pthread_init_static_tls;

> -

>    /* Register the fork generation counter with the libc.  */

>  #ifndef TLS_MULTIPLE_THREADS_IN_TCB

>    __libc_multiple_threads_ptr =

> diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h

> index 00d2cfe764..0ce63672c4 100644

> --- a/nptl/pthreadP.h

> +++ b/nptl/pthreadP.h

> @@ -381,8 +381,6 @@ extern int __pthread_multiple_threads attribute_hidden;

>  extern int *__libc_multiple_threads_ptr attribute_hidden;

>  #endif

>

> -extern void __pthread_init_static_tls (struct link_map *) attribute_hidden;

> -

>  extern size_t __pthread_get_minstack (const pthread_attr_t *attr);

>

>  /* Namespace save aliases.  */

> diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h

> index 67c6686015..1b064c5894 100644

> --- a/sysdeps/generic/ldsodefs.h

> +++ b/sysdeps/generic/ldsodefs.h

> @@ -464,7 +464,9 @@ struct rtld_global

>    /* Generation counter for the dtv.  */

>    EXTERN size_t _dl_tls_generation;

>

> +#if !THREAD_GSCOPE_IN_TCB

>    EXTERN void (*_dl_init_static_tls) (struct link_map *);

> +#endif

>

>    /* Scopes to free after next THREAD_GSCOPE_WAIT ().  */

>    EXTERN struct dl_scope_free_list

> @@ -1270,6 +1272,23 @@ extern void _dl_non_dynamic_init (void)

>  extern void _dl_aux_init (ElfW(auxv_t) *av)

>       attribute_hidden;

>

> +/* Initialize the static TLS space for the link map in all existing

> +   threads. */

> +#if THREAD_GSCOPE_IN_TCB

> +void _dl_init_static_tls (struct link_map *map) attribute_hidden;

> +#endif

> +static inline void

> +dl_init_static_tls (struct link_map *map)

> +{

> +#if THREAD_GSCOPE_IN_TCB

> +  /* The stack list is available to ld.so, so the initialization can

> +     be handled within ld.so directly.  */

> +  _dl_init_static_tls (map);

> +#else

> +  GL (dl_init_static_tls) (map);

> +#endif

> +}

> +

>  /* Return true if the ld.so copy in this namespace is actually active

>     and working.  If false, the dl_open/dlfcn hooks have to be used to

>     call into the outer dynamic linker (which happens after static

>


LGTM.

Thanks.

-- 
H.J.

Patch

diff --git a/elf/dl-open.c b/elf/dl-open.c
index ab7aaa345e..09f0df7d38 100644
--- a/elf/dl-open.c
+++ b/elf/dl-open.c
@@ -426,7 +426,7 @@  TLS generation counter wrapped!  Please report this."));
 	  _dl_update_slotinfo (imap->l_tls_modid);
 #endif
 
-	  GL(dl_init_static_tls) (imap);
+	  dl_init_static_tls (imap);
 	  assert (imap->l_need_tls_init == 0);
 	}
     }
diff --git a/elf/dl-reloc.c b/elf/dl-reloc.c
index c2df26deea..bb9ca1a101 100644
--- a/elf/dl-reloc.c
+++ b/elf/dl-reloc.c
@@ -118,7 +118,7 @@  _dl_try_allocate_static_tls (struct link_map *map, bool optional)
 	(void) _dl_update_slotinfo (map->l_tls_modid);
 #endif
 
-      GL(dl_init_static_tls) (map);
+      dl_init_static_tls (map);
     }
   else
     map->l_need_tls_init = 1;
@@ -141,6 +141,7 @@  cannot allocate memory in static TLS block"));
     }
 }
 
+#if !THREAD_GSCOPE_IN_TCB
 /* Initialize static TLS area and DTV for current (only) thread.
    libpthread implementations should provide their own hook
    to handle all threads.  */
@@ -159,7 +160,7 @@  _dl_nothread_init_static_tls (struct link_map *map)
   memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
 	  '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
 }
-
+#endif /* !THREAD_GSCOPE_IN_TCB */
 
 void
 _dl_relocate_object (struct link_map *l, struct r_scope_elem *scope[],
diff --git a/elf/dl-support.c b/elf/dl-support.c
index 7fc2ee78e2..f966a2e7cd 100644
--- a/elf/dl-support.c
+++ b/elf/dl-support.c
@@ -138,8 +138,6 @@  void *_dl_random;
 #include <dl-procruntime.c>
 #include <dl-procinfo.c>
 
-void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;
-
 size_t _dl_pagesize = EXEC_PAGESIZE;
 
 size_t _dl_minsigstacksize = CONSTANT_MINSIGSTKSZ;
@@ -197,6 +195,7 @@  list_t _dl_stack_user;
 int _dl_stack_cache_lock;
 #else
 int _dl_thread_gscope_count;
+void (*_dl_init_static_tls) (struct link_map *) = &_dl_nothread_init_static_tls;
 #endif
 struct dl_scope_free_list *_dl_scope_free_list;
 
diff --git a/elf/dl-tls.c b/elf/dl-tls.c
index f8b32b3ecb..6baff0c1ea 100644
--- a/elf/dl-tls.c
+++ b/elf/dl-tls.c
@@ -29,6 +29,10 @@ 
 #include <dl-tls.h>
 #include <ldsodefs.h>
 
+#if THREAD_GSCOPE_IN_TCB
+# include <list.h>
+#endif
+
 #define TUNABLE_NAMESPACE rtld
 #include <dl-tunables.h>
 
@@ -1005,3 +1009,38 @@  cannot create TLS data structures"));
       listp->slotinfo[idx].gen = GL(dl_tls_generation) + 1;
     }
 }
+
+#if THREAD_GSCOPE_IN_TCB
+static inline void __attribute__((always_inline))
+init_one_static_tls (struct pthread *curp, struct link_map *map)
+{
+# if TLS_TCB_AT_TP
+  void *dest = (char *) curp - map->l_tls_offset;
+# elif TLS_DTV_AT_TP
+  void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
+# else
+#  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
+# endif
+
+  /* Initialize the memory.  */
+  memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
+	  '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
+}
+
+void
+_dl_init_static_tls (struct link_map *map)
+{
+  lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
+
+  /* Iterate over the list with system-allocated threads first.  */
+  list_t *runp;
+  list_for_each (runp, &GL (dl_stack_used))
+    init_one_static_tls (list_entry (runp, struct pthread, list), map);
+
+  /* Now the list with threads using user-allocated stacks.  */
+  list_for_each (runp, &GL (dl_stack_user))
+    init_one_static_tls (list_entry (runp, struct pthread, list), map);
+
+  lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
+}
+#endif /* THREAD_GSCOPE_IN_TCB */
diff --git a/elf/rtld.c b/elf/rtld.c
index 34879016ad..ad325d4c10 100644
--- a/elf/rtld.c
+++ b/elf/rtld.c
@@ -1139,7 +1139,9 @@  dl_main (const ElfW(Phdr) *phdr,
   struct dl_main_state state;
   dl_main_state_init (&state);
 
+#if !THREAD_GSCOPE_IN_TCB
   GL(dl_init_static_tls) = &_dl_nothread_init_static_tls;
+#endif
 
 #if defined SHARED && defined _LIBC_REENTRANT \
     && defined __rtld_lock_default_lock_recursive
diff --git a/nptl/allocatestack.c b/nptl/allocatestack.c
index f1270c3109..8aaba088b1 100644
--- a/nptl/allocatestack.c
+++ b/nptl/allocatestack.c
@@ -951,38 +951,3 @@  __reclaim_stacks (void)
   GL (dl_stack_cache_lock) = LLL_LOCK_INITIALIZER;
   __default_pthread_attr_lock = LLL_LOCK_INITIALIZER;
 }
-
-
-static inline void __attribute__((always_inline))
-init_one_static_tls (struct pthread *curp, struct link_map *map)
-{
-# if TLS_TCB_AT_TP
-  void *dest = (char *) curp - map->l_tls_offset;
-# elif TLS_DTV_AT_TP
-  void *dest = (char *) curp + map->l_tls_offset + TLS_PRE_TCB_SIZE;
-# else
-#  error "Either TLS_TCB_AT_TP or TLS_DTV_AT_TP must be defined"
-# endif
-
-  /* Initialize the memory.  */
-  memset (__mempcpy (dest, map->l_tls_initimage, map->l_tls_initimage_size),
-	  '\0', map->l_tls_blocksize - map->l_tls_initimage_size);
-}
-
-void
-attribute_hidden
-__pthread_init_static_tls (struct link_map *map)
-{
-  lll_lock (GL (dl_stack_cache_lock), LLL_PRIVATE);
-
-  /* Iterate over the list with system-allocated threads first.  */
-  list_t *runp;
-  list_for_each (runp, &GL (dl_stack_used))
-    init_one_static_tls (list_entry (runp, struct pthread, list), map);
-
-  /* Now the list with threads using user-allocated stacks.  */
-  list_for_each (runp, &GL (dl_stack_user))
-    init_one_static_tls (list_entry (runp, struct pthread, list), map);
-
-  lll_unlock (GL (dl_stack_cache_lock), LLL_PRIVATE);
-}
diff --git a/nptl/nptl-init.c b/nptl/nptl-init.c
index b0879bd87e..fcab5a0904 100644
--- a/nptl/nptl-init.c
+++ b/nptl/nptl-init.c
@@ -191,8 +191,6 @@  __pthread_initialize_minimal_internal (void)
   GL(dl_make_stack_executable_hook) = &__make_stacks_executable;
 #endif
 
-  GL(dl_init_static_tls) = &__pthread_init_static_tls;
-
   /* Register the fork generation counter with the libc.  */
 #ifndef TLS_MULTIPLE_THREADS_IN_TCB
   __libc_multiple_threads_ptr =
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 00d2cfe764..0ce63672c4 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -381,8 +381,6 @@  extern int __pthread_multiple_threads attribute_hidden;
 extern int *__libc_multiple_threads_ptr attribute_hidden;
 #endif
 
-extern void __pthread_init_static_tls (struct link_map *) attribute_hidden;
-
 extern size_t __pthread_get_minstack (const pthread_attr_t *attr);
 
 /* Namespace save aliases.  */
diff --git a/sysdeps/generic/ldsodefs.h b/sysdeps/generic/ldsodefs.h
index 67c6686015..1b064c5894 100644
--- a/sysdeps/generic/ldsodefs.h
+++ b/sysdeps/generic/ldsodefs.h
@@ -464,7 +464,9 @@  struct rtld_global
   /* Generation counter for the dtv.  */
   EXTERN size_t _dl_tls_generation;
 
+#if !THREAD_GSCOPE_IN_TCB
   EXTERN void (*_dl_init_static_tls) (struct link_map *);
+#endif
 
   /* Scopes to free after next THREAD_GSCOPE_WAIT ().  */
   EXTERN struct dl_scope_free_list
@@ -1270,6 +1272,23 @@  extern void _dl_non_dynamic_init (void)
 extern void _dl_aux_init (ElfW(auxv_t) *av)
      attribute_hidden;
 
+/* Initialize the static TLS space for the link map in all existing
+   threads. */
+#if THREAD_GSCOPE_IN_TCB
+void _dl_init_static_tls (struct link_map *map) attribute_hidden;
+#endif
+static inline void
+dl_init_static_tls (struct link_map *map)
+{
+#if THREAD_GSCOPE_IN_TCB
+  /* The stack list is available to ld.so, so the initialization can
+     be handled within ld.so directly.  */
+  _dl_init_static_tls (map);
+#else
+  GL (dl_init_static_tls) (map);
+#endif
+}
+
 /* Return true if the ld.so copy in this namespace is actually active
    and working.  If false, the dl_open/dlfcn hooks have to be used to
    call into the outer dynamic linker (which happens after static