threads: Adds the ability to set a thread priority

Signed-off-by: Ali Beyad <abeyad@google.com>

envoy/thread/thread.h

@@ -55,7 +55,16 @@ using ThreadPtr = std::unique_ptr<Thread>;
 
 // Options specified during thread creation.
 struct Options {
-  std::string name_; // A name supplied for the thread. On Linux this is limited to 15 chars.
+  // A name supplied for the thread. On Linux this is limited to 15 chars.
+  std::string name_;
+  // An optional thread priority for the thread. The value will mean different things on different
+  // platforms. For example, on Linux or Android, the values can range from -20 to 19. On Apple
+  // platforms, the value can range from 1 to 100, which is used to divide by 100 to get a [0,1]
+  // value that can be used on Apple's NSThread.setThreadPriority method.
+  //
+  // If no value is set, the thread will be created with the default thread priority for the
+  // platform.
+  absl::optional<int> priority_{absl::nullopt};
 };
 
 using OptionsOptConstRef = const absl::optional<Options>&;

mobile/library/common/internal_engine.cc

@@ -95,20 +95,12 @@ envoy_status_t InternalEngine::cancelStream(envoy_stream_t stream) {
 // copy-constructible type, so it's not possible to move capture `std::unique_ptr` with
 // `std::function`.
 envoy_status_t InternalEngine::run(std::shared_ptr<Envoy::OptionsImplBase> options) {
-  main_thread_ = thread_factory_->createThread(
-      [this, options]() mutable -> void {
-        if (thread_priority_) {
-          // Set the thread priority before invoking the thread routine.
-          const int rc = setpriority(PRIO_PROCESS, thread_factory_->currentThreadId().getId(),
-                                     *thread_priority_);
-          if (rc != 0) {
-            ENVOY_LOG(debug, "failed to set thread priority: {}", Envoy::errorDetails(errno));
-          }
-        }
-
-        main(options);
-      },
-      /* options= */ absl::nullopt, /* crash_on_failure= */ false);
+  Thread::Options thread_options;
+  if (thread_priority_) {
+    thread_options.priority_ = *thread_priority_;
+  }
+  main_thread_ = thread_factory_->createThread([this, options]() mutable -> void { main(options); },
+                                               thread_options, /* crash_on_failure= */ false);
   return (main_thread_ != nullptr) ? ENVOY_SUCCESS : ENVOY_FAILURE;
 }
 

mobile/test/common/BUILD

@@ -18,6 +18,7 @@ envoy_cc_test(
 envoy_cc_test(
     name = "internal_engine_test",
     srcs = ["internal_engine_test.cc"],
+    copts = ["-Wno-old-style-cast"],
     repository = "@envoy",
     deps = [
         "//library/cc:engine_builder_lib",

mobile/test/common/internal_engine_test.cc

@@ -20,6 +20,14 @@
 #include "library/common/http/header_utility.h"
 #include "library/common/internal_engine.h"
 
+#if defined(__linux__)
+#include <sys/resource.h>
+#include <sys/syscall.h>
+#elif defined(__APPLE__)
+#include <objc/message.h>
+#include <objc/runtime.h>
+#endif
+
 namespace Envoy {
 
 using testing::_;
@@ -96,6 +104,25 @@ Http::ResponseHeaderMapPtr toResponseHeaders(envoy_headers headers) {
   return transformed_headers;
 }
 
+int getThreadPriority() {
+#if defined(__linux__)
+  return getpriority(PRIO_PROCESS, 0);
+#elif defined(__APPLE__)
+  Class nsthread = objc_getClass("NSThread");
+  if (nsthread != nullptr) {
+    SEL selector = sel_registerName("threadPriority");
+    double thread_priority = ((double (*)(Class, SEL))objc_msgSend)(nsthread, selector);
+    // When setting the thread priority, Apple sometimes changes it slightly internally, e.g.
+    // setting to 0.6 can end up as 0.5976. Hence, the rounding enables verifying the values
+    // easier.
+    return std::round(thread_priority * 100);
+  }
+  return 0;
+#else
+  return 0;
+#endif
+}
+
 class InternalEngineTest : public testing::Test {
 public:
   void SetUp() override {
@@ -476,7 +503,7 @@ class ThreadPriorityInternalEngineTest : public InternalEngineTest {
     stream_callbacks.on_headers_ = [&](const Http::ResponseHeaderMap&, bool /* end_stream */,
                                        envoy_stream_intel) {
       // Gets the thread priority, so we can check that it's the same thread priority we set.
-      context.thread_priority = getpriority(PRIO_PROCESS, 0);
+      context.thread_priority = getThreadPriority();
     };
     stream_callbacks.on_complete_ = [&](envoy_stream_intel, envoy_final_stream_intel) {
       context.on_complete_notification.Notify();
@@ -496,15 +523,11 @@ class ThreadPriorityInternalEngineTest : public InternalEngineTest {
   }
 };
 
-// The setpriority() call fails on some Apple environments.
-// TODO(abeyad): investigate what to do for Apple.
-#ifndef __APPLE__
 TEST_F(ThreadPriorityInternalEngineTest, SetThreadPriority) {
   const int expected_thread_priority = 10;
   const int actual_thread_priority = startEngineWithPriority(expected_thread_priority);
   EXPECT_EQ(actual_thread_priority, expected_thread_priority);
 }
-#endif
 
 TEST_F(ThreadPriorityInternalEngineTest, SetOutOfRangeThreadPriority) {
   // 42 is outside the range of acceptable thread priorities.

source/common/common/BUILD

@@ -429,10 +429,12 @@ envoy_cc_posix_library(
     name = "thread_impl_lib",
     srcs = ["posix/thread_impl.cc"],
     hdrs = ["posix/thread_impl.h"],
+    copts = ["-Wno-old-style-cast"],
     include_prefix = "source/common/common",
     strip_include_prefix = "posix",
     deps = [
         ":assert_lib",
+        ":utility_lib",
         "//envoy/thread:thread_interface",
     ],
 )

source/common/common/posix/thread_impl.cc

@@ -3,11 +3,16 @@
 #include "envoy/thread/thread.h"
 
 #include "source/common/common/assert.h"
+#include "source/common/common/utility.h"
 
 #include "absl/strings/str_cat.h"
 
 #if defined(__linux__)
+#include <sys/resource.h>
 #include <sys/syscall.h>
+#elif defined(__APPLE__)
+#include <objc/message.h>
+#include <objc/runtime.h>
 #endif
 
 namespace Envoy {
@@ -34,18 +39,44 @@ int64_t getCurrentThreadId() {
   return tid;
 }
 
+void setThreadPriority(const int64_t tid, const int priority) {
+#if defined(__linux__)
+  const int rc = setpriority(PRIO_PROCESS, tid, priority);
+  if (rc != 0) {
+    ENVOY_LOG_MISC(warn, "failed to set thread priority: {}", Envoy::errorDetails(errno));
+  }
+#elif defined(__APPLE__)
+  UNREFERENCED_PARAMETER(tid);
+  // Use NSThread via the Objective-C runtime to set the thread priority; it's the best way to set
+  // the thread priority on Apple platforms, and directly invoking setpriority() on iOS fails with
+  // permissions issues, as discovered through manual testing.
+  Class nsthread = objc_getClass("NSThread");
+  if (nsthread != nullptr) {
+    id (*getCurrentNSThread)(Class, SEL) = (id(*)(Class, SEL))objc_msgSend;
+    id current_thread = getCurrentNSThread(nsthread, sel_registerName("currentThread"));
+    void (*setNSThreadPriority)(id, SEL, double) = (void (*)(id, SEL, double))objc_msgSend;
+    setNSThreadPriority(current_thread, sel_registerName("setThreadPriority:"), priority);
+  }
+#else
+#error "Enable and test pthread id retrieval code for you arch in pthread/thread_impl.cc"
+#endif
+}
+
 } // namespace
 
 // See https://www.man7.org/linux/man-pages/man3/pthread_setname_np.3.html.
 // The maximum thread name is 16 bytes including the terminating nul byte,
 // so we need to truncate the string_view to 15 bytes.
 #define PTHREAD_MAX_THREADNAME_LEN_INCLUDING_NULL_BYTE 16
 
-ThreadHandle::ThreadHandle(std::function<void()> thread_routine)
-    : thread_routine_(thread_routine) {}
+ThreadHandle::ThreadHandle(std::function<void()> thread_routine,
+                           const absl::optional<int>& thread_priority)
+    : thread_routine_(thread_routine), thread_priority_(thread_priority) {}
 
 /** Returns the thread routine. */
-std::function<void()>& ThreadHandle::routine() { return thread_routine_; };
+std::function<void()>& ThreadHandle::routine() { return thread_routine_; }
+
+const absl::optional<int>& ThreadHandle::priority() const { return thread_priority_; }
 
 /** Returns the thread handle. */
 pthread_t& ThreadHandle::handle() { return thread_handle_; }
@@ -140,15 +171,20 @@ int PosixThreadFactory::createPthread(ThreadHandle* thread_handle) {
   return pthread_create(
       &thread_handle->handle(), nullptr,
       [](void* arg) -> void* {
-        static_cast<ThreadHandle*>(arg)->routine()();
+        ThreadHandle* handle = static_cast<ThreadHandle*>(arg);
+        if (handle->priority()) {
+          setThreadPriority(getCurrentThreadId(), *handle->priority());
+        }
+        handle->routine()();
         return nullptr;
       },
       reinterpret_cast<void*>(thread_handle));
 }
 
 PosixThreadPtr PosixThreadFactory::createThread(std::function<void()> thread_routine,
                                                 OptionsOptConstRef options, bool crash_on_failure) {
-  auto thread_handle = new ThreadHandle(thread_routine);
+  auto thread_handle =
+      new ThreadHandle(thread_routine, options ? options->priority_ : absl::nullopt);
   const int rc = createPthread(thread_handle);
   if (rc != 0) {
     delete thread_handle;

source/common/common/posix/thread_impl.h

@@ -12,16 +12,20 @@ namespace Thread {
 
 class ThreadHandle {
 public:
-  explicit ThreadHandle(std::function<void()> thread_routine);
+  ThreadHandle(std::function<void()> thread_routine, const absl::optional<int>& thread_priority);
 
   /** Returns the thread routine. */
   std::function<void()>& routine();
 
+  /** Returns the thread priority, if any. */
+  const absl::optional<int>& priority() const;
+
   /** Returns the thread handle. */
   pthread_t& handle();
 
 private:
   std::function<void()> thread_routine_;
+  const absl::optional<int> thread_priority_;
   pthread_t thread_handle_;
 };
 

source/common/common/win32/thread_impl.cc

@@ -22,6 +22,10 @@ ThreadImplWin32::ThreadImplWin32(std::function<void()> thread_routine, OptionsOp
         return 0;
       },
       this, 0, nullptr));
+  if (options && options.thread_priority_ &&
+      !SetThreadPriority(thread_handle_, *options.thread_priority_)) {
+    ENVOY_LOG_MISC(warn, "Could not set the thread priority to {}", *options.thread_priority_);
+  }
   RELEASE_ASSERT(thread_handle_ != 0, "");
 }