Implement a stub pthreads library for THREAD_MODEL=single (#518)

~~This patch series first starts with a number of commits stubbing out
functions in the existing `THREAD_model=posix` code. According to "The
Open Group Base Specifications Issue 7, 2018 edition", there are a
number of mandatory functions which have not been provided. There are
also some optional functions that have been partially provided in a
not-useful way (e.g. get but no set function). For these, I have chosen
to clean them up and remove the get functions for consistency.~~ EDIT:
These have been split off into separate PRs and merged.

The remainder of the patches then build up a stub implementation of
pthreads for `THREAD_MODEL=single`. I have done my best to try to make
sure that all functions are as conforming as possible (under the
assumption that another thread cannot ever be launched). This means that
objects such as mutexes and rwlocks actually do update their state and
will correctly fail when locks cannot be acquired.

When an inevitable deadlock occurs, I have chosen to return EDEADLK when
it has been explicitly listed as a permissible return value, and to
invoke `__builtin_trap` otherwise.

I have tested this by rebuilding libc++ with threads enabled and then
smoke-testing Clang/LLVM-on-WASI to make sure that it can compile a
simple program. I have not run any more-extensive conformance testing.

Fixes #501

Makefile

@@ -76,6 +76,7 @@ DLMALLOC_SOURCES = $(DLMALLOC_SRC_DIR)/dlmalloc.c
 DLMALLOC_INC = $(DLMALLOC_DIR)/include
 EMMALLOC_DIR = emmalloc
 EMMALLOC_SOURCES = $(EMMALLOC_DIR)/emmalloc.c
+STUB_PTHREADS_DIR = stub-pthreads
 LIBC_BOTTOM_HALF_DIR = libc-bottom-half
 LIBC_BOTTOM_HALF_CLOUDLIBC_SRC = $(LIBC_BOTTOM_HALF_DIR)/cloudlibc/src
 LIBC_BOTTOM_HALF_CLOUDLIBC_SRC_INC = $(LIBC_BOTTOM_HALF_CLOUDLIBC_SRC)/include
@@ -142,6 +143,8 @@ LIBWASI_EMULATED_SIGNAL_SOURCES = \
 LIBWASI_EMULATED_SIGNAL_MUSL_SOURCES = \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/signal/psignal.c \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/string/strsignal.c
+LIBWASI_EMULATED_PTHREAD_SOURCES = \
+    $(STUB_PTHREADS_DIR)/stub-pthreads-emulated.c
 LIBDL_SOURCES = $(LIBC_TOP_HALF_MUSL_SRC_DIR)/misc/dl.c
 LIBSETJMP_SOURCES = $(LIBC_TOP_HALF_MUSL_SRC_DIR)/setjmp/wasm32/rt.c
 LIBC_BOTTOM_HALF_CRT_SOURCES = $(wildcard $(LIBC_BOTTOM_HALF_DIR)/crt/*.c)
@@ -274,88 +277,93 @@ LIBC_TOP_HALF_MUSL_SOURCES += \
     )
 endif
 
-ifeq ($(THREAD_MODEL), posix)
+# pthreads functions (possibly stub) for either thread model
 LIBC_TOP_HALF_MUSL_SOURCES += \
     $(addprefix $(LIBC_TOP_HALF_MUSL_SRC_DIR)/, \
-        env/__init_tls.c \
-        stdio/__lockfile.c \
-        stdio/flockfile.c \
-        stdio/ftrylockfile.c \
-        stdio/funlockfile.c \
-        thread/__lock.c \
-        thread/__wait.c \
-        thread/__timedwait.c \
         thread/default_attr.c \
         thread/pthread_attr_destroy.c \
         thread/pthread_attr_get.c \
         thread/pthread_attr_init.c \
         thread/pthread_attr_setdetachstate.c \
         thread/pthread_attr_setguardsize.c \
+        thread/pthread_attr_setschedparam.c \
         thread/pthread_attr_setstack.c \
         thread/pthread_attr_setstacksize.c \
-        thread/pthread_attr_setschedparam.c \
-        thread/pthread_barrier_destroy.c \
-        thread/pthread_barrier_init.c \
-        thread/pthread_barrier_wait.c \
         thread/pthread_barrierattr_destroy.c \
         thread/pthread_barrierattr_init.c \
         thread/pthread_barrierattr_setpshared.c \
-        thread/pthread_cleanup_push.c \
         thread/pthread_cancel.c \
-        thread/pthread_cond_broadcast.c \
-        thread/pthread_cond_destroy.c \
-        thread/pthread_cond_init.c \
-        thread/pthread_cond_signal.c \
-        thread/pthread_cond_timedwait.c \
-        thread/pthread_cond_wait.c \
+        thread/pthread_cleanup_push.c \
         thread/pthread_condattr_destroy.c \
         thread/pthread_condattr_init.c \
         thread/pthread_condattr_setclock.c \
         thread/pthread_condattr_setpshared.c \
-        thread/pthread_create.c \
-        thread/pthread_detach.c \
         thread/pthread_equal.c \
-        thread/pthread_getattr_np.c \
         thread/pthread_getspecific.c \
-        thread/pthread_join.c \
         thread/pthread_key_create.c \
-        thread/pthread_mutex_consistent.c \
         thread/pthread_mutex_destroy.c \
         thread/pthread_mutex_init.c \
-        thread/pthread_mutex_getprioceiling.c \
-        thread/pthread_mutex_lock.c \
-        thread/pthread_mutex_timedlock.c \
-        thread/pthread_mutex_trylock.c \
-        thread/pthread_mutex_unlock.c \
         thread/pthread_mutexattr_destroy.c \
         thread/pthread_mutexattr_init.c \
         thread/pthread_mutexattr_setprotocol.c \
         thread/pthread_mutexattr_setpshared.c \
         thread/pthread_mutexattr_setrobust.c \
         thread/pthread_mutexattr_settype.c \
-        thread/pthread_once.c \
         thread/pthread_rwlock_destroy.c \
         thread/pthread_rwlock_init.c \
-        thread/pthread_rwlock_rdlock.c \
-        thread/pthread_rwlock_timedrdlock.c \
-        thread/pthread_rwlock_timedwrlock.c \
-        thread/pthread_rwlock_tryrdlock.c \
-        thread/pthread_rwlock_trywrlock.c \
-        thread/pthread_rwlock_unlock.c \
-        thread/pthread_rwlock_wrlock.c \
         thread/pthread_rwlockattr_destroy.c \
         thread/pthread_rwlockattr_init.c \
         thread/pthread_rwlockattr_setpshared.c \
+        thread/pthread_self.c \
         thread/pthread_setcancelstate.c \
         thread/pthread_setcanceltype.c \
         thread/pthread_setspecific.c \
-        thread/pthread_self.c \
         thread/pthread_spin_destroy.c \
         thread/pthread_spin_init.c \
+        thread/pthread_testcancel.c \
+    )
+ifeq ($(THREAD_MODEL), posix)
+# pthreads functions needed for actual thread support
+LIBC_TOP_HALF_MUSL_SOURCES += \
+    $(addprefix $(LIBC_TOP_HALF_MUSL_SRC_DIR)/, \
+        env/__init_tls.c \
+        stdio/__lockfile.c \
+        stdio/flockfile.c \
+        stdio/ftrylockfile.c \
+        stdio/funlockfile.c \
+        thread/__lock.c \
+        thread/__wait.c \
+        thread/__timedwait.c \
+        thread/pthread_barrier_destroy.c \
+        thread/pthread_barrier_init.c \
+        thread/pthread_barrier_wait.c \
+        thread/pthread_cond_broadcast.c \
+        thread/pthread_cond_destroy.c \
+        thread/pthread_cond_init.c \
+        thread/pthread_cond_signal.c \
+        thread/pthread_cond_timedwait.c \
+        thread/pthread_cond_wait.c \
+        thread/pthread_create.c \
+        thread/pthread_detach.c \
+        thread/pthread_getattr_np.c \
+        thread/pthread_join.c \
+        thread/pthread_mutex_consistent.c \
+        thread/pthread_mutex_getprioceiling.c \
+        thread/pthread_mutex_lock.c \
+        thread/pthread_mutex_timedlock.c \
+        thread/pthread_mutex_trylock.c \
+        thread/pthread_mutex_unlock.c \
+        thread/pthread_once.c \
+        thread/pthread_rwlock_rdlock.c \
+        thread/pthread_rwlock_timedrdlock.c \
+        thread/pthread_rwlock_timedwrlock.c \
+        thread/pthread_rwlock_tryrdlock.c \
+        thread/pthread_rwlock_trywrlock.c \
+        thread/pthread_rwlock_unlock.c \
+        thread/pthread_rwlock_wrlock.c \
         thread/pthread_spin_lock.c \
         thread/pthread_spin_trylock.c \
         thread/pthread_spin_unlock.c \
-        thread/pthread_testcancel.c \
         thread/sem_destroy.c \
         thread/sem_getvalue.c \
         thread/sem_init.c \
@@ -366,6 +374,16 @@ LIBC_TOP_HALF_MUSL_SOURCES += \
         thread/wasm32/wasi_thread_start.s \
     )
 endif
+ifeq ($(THREAD_MODEL), single)
+# pthreads stubs for single-threaded environment
+LIBC_TOP_HALF_MUSL_SOURCES += \
+    $(STUB_PTHREADS_DIR)/barrier.c \
+    $(STUB_PTHREADS_DIR)/condvar.c \
+    $(STUB_PTHREADS_DIR)/mutex.c \
+    $(STUB_PTHREADS_DIR)/rwlock.c \
+    $(STUB_PTHREADS_DIR)/spinlock.c \
+    $(STUB_PTHREADS_DIR)/stub-pthreads-good.c
+endif
 
 MUSL_PRINTSCAN_SOURCES = \
     $(LIBC_TOP_HALF_MUSL_SRC_DIR)/internal/floatscan.c \
@@ -418,10 +436,10 @@ ifeq ($(THREAD_MODEL), posix)
 # https://reviews.llvm.org/D130053).
 CFLAGS += -mthread-model posix -pthread -ftls-model=local-exec
 ASMFLAGS += -matomics
+endif
 
 # Include cloudlib's directory to access the structure definition of clockid_t
 CFLAGS += -I$(LIBC_BOTTOM_HALF_CLOUDLIBC_SRC)
-endif
 
 ifneq ($(LTO),no)
 ifeq ($(LTO),full)
@@ -490,6 +508,7 @@ LIBWASI_EMULATED_PROCESS_CLOCKS_OBJS = $(call objs,$(LIBWASI_EMULATED_PROCESS_CL
 LIBWASI_EMULATED_GETPID_OBJS = $(call objs,$(LIBWASI_EMULATED_GETPID_SOURCES))
 LIBWASI_EMULATED_SIGNAL_OBJS = $(call objs,$(LIBWASI_EMULATED_SIGNAL_SOURCES))
 LIBWASI_EMULATED_SIGNAL_MUSL_OBJS = $(call objs,$(LIBWASI_EMULATED_SIGNAL_MUSL_SOURCES))
+LIBWASI_EMULATED_PTHREAD_OBJS = $(call objs,$(LIBWASI_EMULATED_PTHREAD_SOURCES))
 LIBDL_OBJS = $(call objs,$(LIBDL_SOURCES))
 LIBSETJMP_OBJS = $(call objs,$(LIBSETJMP_SOURCES))
 LIBC_BOTTOM_HALF_CRT_OBJS = $(call objs,$(LIBC_BOTTOM_HALF_CRT_SOURCES))
@@ -593,6 +612,7 @@ LIBWASI_EMULATED_PROCESS_CLOCKS_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULA
 LIBWASI_EMULATED_GETPID_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULATED_GETPID_OBJS))
 LIBWASI_EMULATED_SIGNAL_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULATED_SIGNAL_OBJS))
 LIBWASI_EMULATED_SIGNAL_MUSL_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULATED_SIGNAL_MUSL_OBJS))
+LIBWASI_EMULATED_PTHREAD_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBWASI_EMULATED_PTHREAD_OBJS))
 LIBDL_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBDL_OBJS))
 LIBSETJMP_SO_OBJS = $(patsubst %.o,%.pic.o,$(LIBSETJMP_OBJS))
 BULK_MEMORY_SO_OBJS = $(patsubst %.o,%.pic.o,$(BULK_MEMORY_OBJS))
@@ -608,6 +628,7 @@ PIC_OBJS = \
 	$(LIBWASI_EMULATED_GETPID_SO_OBJS) \
 	$(LIBWASI_EMULATED_SIGNAL_SO_OBJS) \
 	$(LIBWASI_EMULATED_SIGNAL_MUSL_SO_OBJS) \
+	$(LIBWASI_EMULATED_PTHREAD_SO_OBJS) \
 	$(LIBDL_SO_OBJS) \
 	$(LIBSETJMP_SO_OBJS) \
 	$(BULK_MEMORY_SO_OBJS) \
@@ -649,6 +670,8 @@ $(OBJDIR)/libwasi-emulated-getpid.so.a: $(LIBWASI_EMULATED_GETPID_SO_OBJS)
 
 $(OBJDIR)/libwasi-emulated-signal.so.a: $(LIBWASI_EMULATED_SIGNAL_SO_OBJS) $(LIBWASI_EMULATED_SIGNAL_MUSL_SO_OBJS)
 
+$(OBJDIR)/libwasi-emulated-pthread.so.a: $(LIBWASI_EMULATED_PTHREAD_SO_OBJS)
+
 $(OBJDIR)/libdl.so.a: $(LIBDL_SO_OBJS)
 
 $(OBJDIR)/libsetjmp.so.a: $(LIBSETJMP_SO_OBJS)
@@ -667,6 +690,8 @@ $(SYSROOT_LIB)/libwasi-emulated-getpid.a: $(LIBWASI_EMULATED_GETPID_OBJS)
 
 $(SYSROOT_LIB)/libwasi-emulated-signal.a: $(LIBWASI_EMULATED_SIGNAL_OBJS) $(LIBWASI_EMULATED_SIGNAL_MUSL_OBJS)
 
+$(SYSROOT_LIB)/libwasi-emulated-pthread.a: $(LIBWASI_EMULATED_PTHREAD_OBJS)
+
 $(SYSROOT_LIB)/libdl.a: $(LIBDL_OBJS)
 
 $(SYSROOT_LIB)/libsetjmp.a: $(LIBSETJMP_OBJS)
@@ -769,6 +794,12 @@ $(FTS_OBJS): CFLAGS += \
 $(LIBWASI_EMULATED_PROCESS_CLOCKS_OBJS) $(LIBWASI_EMULATED_PROCESS_CLOCKS_SO_OBJS): CFLAGS += \
     -I$(LIBC_BOTTOM_HALF_CLOUDLIBC_SRC)
 
+$(LIBWASI_EMULATED_PTHREAD_OBJS) $(LIBWASI_EMULATED_PTHREAD_SO_OBJS): CFLAGS += \
+    -I$(LIBC_TOP_HALF_MUSL_SRC_DIR)/include \
+    -I$(LIBC_TOP_HALF_MUSL_SRC_DIR)/internal \
+    -I$(LIBC_TOP_HALF_MUSL_DIR)/arch/wasm32 \
+    -D_WASI_EMULATED_PTHREAD
+
 # emmalloc uses a lot of pointer type-punning, which is UB under strict aliasing,
 # and this was found to have real miscompilations in wasi-libc#421.
 $(EMMALLOC_OBJS): CFLAGS += \
@@ -820,6 +851,7 @@ LIBC_SO = \
 	$(SYSROOT_LIB)/libwasi-emulated-process-clocks.so \
 	$(SYSROOT_LIB)/libwasi-emulated-getpid.so \
 	$(SYSROOT_LIB)/libwasi-emulated-signal.so \
+	$(SYSROOT_LIB)/libwasi-emulated-pthread.so \
 	$(SYSROOT_LIB)/libdl.so
 ifeq ($(BUILD_LIBSETJMP),yes)
 LIBC_SO += \
@@ -838,6 +870,10 @@ STATIC_LIBS = \
     $(SYSROOT_LIB)/libwasi-emulated-getpid.a \
     $(SYSROOT_LIB)/libwasi-emulated-signal.a \
     $(SYSROOT_LIB)/libdl.a
+ifneq ($(THREAD_MODEL), posix)
+    STATIC_LIBS += \
+        $(SYSROOT_LIB)/libwasi-emulated-pthread.a
+endif
 ifeq ($(BUILD_LIBSETJMP),yes)
 STATIC_LIBS += \
 	$(SYSROOT_LIB)/libsetjmp.a