feat: setup stacks and TLS in wasi_thread_start

and cleanup in `pthread_exit()`

Signed-off-by: Harald Hoyer <harald@profian.com>

Makefile

@@ -190,9 +190,12 @@ LIBC_TOP_HALF_MUSL_SOURCES = \
 ifeq ($(THREAD_MODEL), posix)
 LIBC_TOP_HALF_MUSL_SOURCES += \
     $(addprefix $(LIBC_TOP_HALF_MUSL_SRC_DIR)/, \
+        env/__init_tls.c \
+        stdio/__lockfile.c \
         thread/__lock.c \
         thread/__wait.c \
         thread/__timedwait.c \
+        thread/default_attr.c \
         thread/pthread_cleanup_push.c \
         thread/pthread_cond_broadcast.c \
         thread/pthread_cond_destroy.c \
@@ -233,6 +236,7 @@ LIBC_TOP_HALF_MUSL_SOURCES += \
         thread/pthread_rwlockattr_init.c \
         thread/pthread_rwlockattr_setpshared.c \
         thread/pthread_setcancelstate.c \
+        thread/pthread_self.c \
         thread/pthread_testcancel.c \
         thread/sem_destroy.c \
         thread/sem_getvalue.c \

expected/wasm32-wasi/posix/defined-symbols.txt

@@ -23,6 +23,7 @@ __c_locale
 __clock
 __clock_gettime
 __clock_nanosleep
+__copy_tls
 __cos
 __cosdf
 __cosl
@@ -38,6 +39,8 @@ __ctype_tolower_loc
 __ctype_toupper_loc
 __cxa_atexit
 __cxa_finalize
+__default_guardsize
+__default_stacksize
 __des_setkey
 __do_cleanup_pop
 __do_cleanup_push
@@ -87,6 +90,8 @@ __getopt_msg
 __gmtime_r
 __hwcap
 __inet_aton
+__init_tls
+__init_tp
 __intscan
 __invtrigl_R
 __isalnum_l
@@ -144,6 +149,7 @@ __locale_lock
 __locale_lockptr
 __localtime_r
 __lock
+__lockfile
 __log2_data
 __log2f_data
 __log_data
@@ -265,6 +271,7 @@ __tan
 __tandf
 __tanl
 __testcancel
+__thread_list_lock
 __timedwait
 __timedwait_cp
 __tl_lock
@@ -288,6 +295,7 @@ __tsearch_balance
 __uflow
 __unlist_locked_file
 __unlock
+__unlockfile
 __uselocale
 __utc
 __wait
@@ -371,6 +379,7 @@ __wasilibc_nocwd_scandirat
 __wasilibc_nocwd_symlinkat
 __wasilibc_nocwd_utimensat
 __wasilibc_open_nomode
+__wasilibc_pthread_self
 __wasilibc_register_preopened_fd
 __wasilibc_rename_newat
 __wasilibc_rename_oldat
@@ -992,6 +1001,7 @@ pthread_rwlock_wrlock
 pthread_rwlockattr_destroy
 pthread_rwlockattr_init
 pthread_rwlockattr_setpshared
+pthread_self
 pthread_setcancelstate
 pthread_testcancel
 pthread_timedjoin_np
@@ -1182,6 +1192,7 @@ tfind
 tgamma
 tgammaf
 tgammal
+thrd_current
 thrd_sleep
 time
 timegm

expected/wasm32-wasi/posix/undefined-symbols.txt

@@ -1,7 +1,4 @@
 __addtf3
-__copy_tls
-__default_guardsize
-__default_stacksize
 __divtf3
 __eqtf2
 __extenddftf2
@@ -59,19 +56,18 @@ __imported_wasi_snapshot_preview1_sock_accept
 __imported_wasi_snapshot_preview1_sock_recv
 __imported_wasi_snapshot_preview1_sock_send
 __imported_wasi_snapshot_preview1_sock_shutdown
-__imported_wasi_snapshot_preview2_thread_spawn
+__imported_wasi_thread_spawn
 __letf2
-__lockfile
 __lttf2
 __main_argc_argv
 __netf2
 __stack_pointer
 __subtf3
-__thread_list_lock
+__tls_align
 __tls_base
+__tls_size
 __trunctfdf2
 __trunctfsf2
-__unlockfile
 __unordtf2
-__wasilibc_pthread_self
 __wasm_call_ctors
+__wasm_init_tls

expected/wasm32-wasi/single/defined-symbols.txt

@@ -80,6 +80,7 @@ __getopt_msg
 __gmtime_r
 __hwcap
 __inet_aton
+__init_tls
 __intscan
 __invtrigl_R
 __isalnum_l

libc-bottom-half/crt/crt1-command.c

@@ -1,3 +1,4 @@
+#include "libc.h"
 #include <wasi/api.h>
 extern void __wasm_call_ctors(void);
 extern int __main_void(void);
@@ -7,6 +8,11 @@ extern void __wasm_call_dtors(void);
 // that the `_start` function isn't started more than once.
 static volatile int started = 0;
 
+static void dummy_0()
+{
+}
+weak_alias(dummy_0, __init_tls);
+
 __attribute__((export_name("_start")))
 void _start(void) {
     // Don't allow the program to be called multiple times.
@@ -18,6 +24,8 @@ void _start(void) {
     // The linker synthesizes this to call constructors.
     __wasm_call_ctors();
 
+    __init_tls(NULL);
+
     // Call `__main_void` which will either be the application's zero-argument
     // `__main_void` function or a libc routine which obtains the command-line
     // arguments and calls `__main_argv_argc`.

libc-bottom-half/sources/__wasilibc_real.c

@@ -660,13 +660,13 @@ __wasi_errno_t __wasi_sock_shutdown(
 }
 
 #ifdef _REENTRANT
-int32_t __imported_wasi_snapshot_preview2_thread_spawn(int32_t arg0) __attribute__((
-    __import_module__("wasi_snapshot_preview2"),
+int32_t __imported_wasi_thread_spawn(int32_t arg0) __attribute__((
+    __import_module__("wasi"),
     __import_name__("thread_spawn")
 ));
 
 __wasi_errno_t __wasi_thread_spawn(void* start_arg) {
-	int32_t ret = __imported_wasi_snapshot_preview2_thread_spawn((int32_t) start_arg);
+	int32_t ret = __imported_wasi_thread_spawn((int32_t) start_arg);
     return (uint16_t) ret;
 }
 #endif

libc-top-half/musl/arch/wasm32/pthread_arch.h

@@ -1,11 +1,11 @@
+#include <threads.h>
+
 static inline uintptr_t __get_tp(void) {
 #if _REENTRANT
-  int val;
-  __asm__("global.get __wasilibc_pthread_self\n"
-          "local.set %0"
-          : "=r"(val));
-  return val;
+	extern thread_local uintptr_t __wasilibc_pthread_self;
+	return __wasilibc_pthread_self;
 #else
-  return 0;
+	return 0;
 #endif
 }
+

libc-top-half/musl/src/env/__init_tls.c

@@ -1,13 +1,19 @@
 #define SYSCALL_NO_TLS 1
+#ifdef __wasilibc_unmodified_upstream
 #include <elf.h>
+#endif
 #include <limits.h>
+#ifdef __wasilibc_unmodified_upstream
 #include <sys/mman.h>
+#endif
 #include <string.h>
 #include <stddef.h>
 #include "pthread_impl.h"
 #include "libc.h"
 #include "atomic.h"
+#ifdef __wasilibc_unmodified_upstream
 #include "syscall.h"
+#endif
 
 volatile int __thread_list_lock;
 
@@ -19,7 +25,9 @@ int __init_tp(void *p)
 	if (r < 0) return -1;
 	if (!r) libc.can_do_threads = 1;
 	td->detach_state = DT_JOINABLE;
+#ifdef __wasilibc_unmodified_upstream
 	td->tid = __syscall(SYS_set_tid_address, &__thread_list_lock);
+#endif
 	td->locale = &libc.global_locale;
 	td->robust_list.head = &td->robust_list.head;
 	td->sysinfo = __sysinfo;
@@ -61,31 +69,40 @@ void *__copy_tls(unsigned char *mem)
 	mem -= (uintptr_t)mem & (libc.tls_align-1);
 	td = (pthread_t)mem;
 
+#ifdef __wasilibc_unmodified_upstream
 	for (i=1, p=libc.tls_head; p; i++, p=p->next) {
 		dtv[i] = (uintptr_t)(mem - p->offset) + DTP_OFFSET;
 		memcpy(mem - p->offset, p->image, p->len);
 	}
+#endif
 #endif
 	dtv[0] = libc.tls_cnt;
 	td->dtv = dtv;
 	return td;
 }
 
+#ifdef __wasilibc_unmodified_upstream
 #if ULONG_MAX == 0xffffffff
 typedef Elf32_Phdr Phdr;
 #else
 typedef Elf64_Phdr Phdr;
 #endif
 
 extern weak hidden const size_t _DYNAMIC[];
+#endif
 
+#ifdef __wasilibc_unmodified_upstream
 static void static_init_tls(size_t *aux)
+#else
+void __init_tls(size_t *aux)
+#endif
 {
+	void *mem;
+#ifdef __wasilibc_unmodified_upstream
 	unsigned char *p;
 	size_t n;
 	Phdr *phdr, *tls_phdr=0;
 	size_t base = 0;
-	void *mem;
 
 	for (p=(void *)aux[AT_PHDR],n=aux[AT_PHNUM]; n; n--,p+=aux[AT_PHENT]) {
 		phdr = (void *)p;
@@ -121,7 +138,13 @@ static void static_init_tls(size_t *aux)
 	main_tls.offset = main_tls.size;
 #endif
 	if (main_tls.align < MIN_TLS_ALIGN) main_tls.align = MIN_TLS_ALIGN;
-
+#else // __wasilibc_unmodified_upstream
+	main_tls.size = __builtin_wasm_tls_size();
+	main_tls.offset = main_tls.size;
+	main_tls.align = __builtin_wasm_tls_align();
+	if (main_tls.size > 0)
+		libc.tls_cnt = 1;
+#endif // __wasilibc_unmodified_upstream
 	libc.tls_align = main_tls.align;
 	libc.tls_size = 2*sizeof(void *) + sizeof(struct pthread)
 #ifdef TLS_ABOVE_TP
@@ -130,6 +153,7 @@ static void static_init_tls(size_t *aux)
 		+ main_tls.size + main_tls.align
 		+ MIN_TLS_ALIGN-1 & -MIN_TLS_ALIGN;
 
+#ifdef __wasilibc_unmodified_upstream // FIXME
 	if (libc.tls_size > sizeof builtin_tls) {
 #ifndef SYS_mmap2
 #define SYS_mmap2 SYS_mmap
@@ -144,10 +168,15 @@ static void static_init_tls(size_t *aux)
 	} else {
 		mem = builtin_tls;
 	}
+#else // __wasilibc_unmodified_upstream
+	mem = builtin_tls;
+#endif // __wasilibc_unmodified_upstream
 
 	/* Failure to initialize thread pointer is always fatal. */
 	if (__init_tp(__copy_tls(mem)) < 0)
 		a_crash();
 }
 
+#ifdef __wasilibc_unmodified_upstream
 weak_alias(static_init_tls, __init_tls);
+#endif

libc-top-half/musl/src/internal/pthread_impl.h

@@ -164,7 +164,15 @@ extern hidden void *__pthread_tsd_main[];
 extern hidden volatile int __eintr_valid_flag;
 
 hidden int __clone(int (*)(void *), void *, int, void *, ...);
+#ifdef __wasilibc_unmodified_upstream
 hidden int __set_thread_area(void *);
+#else
+static inline int __set_thread_area(void *p) {
+	extern thread_local uintptr_t __wasilibc_pthread_self;
+	__wasilibc_pthread_self	= (uintptr_t) p;
+	return 0;
+}
+#endif
 #ifdef __wasilibc_unmodified_upstream /* WASI has no sigaction */
 hidden int __libc_sigaction(int, const struct sigaction *, struct sigaction *);
 #endif
@@ -213,8 +221,13 @@ extern hidden volatile int __abort_lock[1];
 extern hidden unsigned __default_stacksize;
 extern hidden unsigned __default_guardsize;
 
+#ifdef __wasilibc_unmodified_upstream
 #define DEFAULT_STACK_SIZE 131072
 #define DEFAULT_GUARD_SIZE 8192
+#else
+#define DEFAULT_STACK_SIZE 131072
+#define DEFAULT_GUARD_SIZE 0
+#endif
 
 #define DEFAULT_STACK_MAX (8<<20)
 #define DEFAULT_GUARD_MAX (1<<20)