issue: flag for adding epfd to inter thread epoll

Signed-off-by: Iftah Levi <iftahl@nvidia.com>

src/core/event/event_handler_manager.cpp

@@ -1053,7 +1053,8 @@ void *event_handler_manager::thread_loop()
             event_handler_map_t::iterator i = m_event_handler_map.find(fd);
             if (i == m_event_handler_map.end()) {
                 // No event handler - this is probably a poll_os event!
-                if (!g_p_fd_collection->set_immediate_os_sample(fd)) {
+                if (safe_mce_sys().os_events_in_internal_thread_epoll &&
+                    !g_p_fd_collection->set_immediate_os_sample(fd)) {
                     evh_logdbg("No event handler (fd=%d)", fd);
                 }
                 continue;

src/core/iomux/epfd_info.cpp

@@ -62,6 +62,7 @@ epfd_info::epfd_info(int epfd, int size)
     , m_lock_poll_os(MULTILOCK_NON_RECURSIVE, "epfd_lock_poll_os")
     , m_sysvar_thread_mode(safe_mce_sys().thread_mode)
     , m_b_os_data_available(false)
+    , use_os_events_in_internal_thread_epoll(safe_mce_sys().os_events_in_internal_thread_epoll)
 {
     __log_funcall("");
     int max_sys_fd = get_sys_max_fd_num();
@@ -89,9 +90,11 @@ epfd_info::epfd_info(int epfd, int size)
 
     xlio_stats_instance_create_epoll_block(m_epfd, &(m_stats->stats));
 
-    // Register this socket to read nonoffloaded data
-    g_p_event_handler_manager->update_epfd(m_epfd, EPOLL_CTL_ADD,
-                                           EPOLLIN | EPOLLPRI | EPOLLONESHOT);
+    if (use_os_events_in_internal_thread_epoll) {
+        // Register this socket to read nonoffloaded data
+        g_p_event_handler_manager->update_epfd(m_epfd, EPOLL_CTL_ADD,
+                                               EPOLLIN | EPOLLPRI | EPOLLONESHOT);
+    }
 
     wakeup_set_epoll_fd(m_epfd);
 }
@@ -133,8 +136,10 @@ epfd_info::~epfd_info()
         BULLSEYE_EXCLUDE_BLOCK_END
     }
 
-    g_p_event_handler_manager->update_epfd(m_epfd, EPOLL_CTL_DEL,
-                                           EPOLLIN | EPOLLPRI | EPOLLONESHOT);
+    if (use_os_events_in_internal_thread_epoll) {
+        g_p_event_handler_manager->update_epfd(m_epfd, EPOLL_CTL_DEL,
+                                               EPOLLIN | EPOLLPRI | EPOLLONESHOT);
+    }
 
     unlock();
 
@@ -864,9 +869,11 @@ void epfd_info::register_to_internal_thread()
     std::lock_guard<decltype(m_lock_poll_os)> locker(m_lock_poll_os);
     m_b_os_data_available = false;
 
-    // Reassign EPOLLIN event
-    g_p_event_handler_manager->update_epfd(m_epfd, EPOLL_CTL_MOD,
-                                           EPOLLIN | EPOLLPRI | EPOLLONESHOT);
+    if (use_os_events_in_internal_thread_epoll) {
+        // Reassign EPOLLIN event
+        g_p_event_handler_manager->update_epfd(m_epfd, EPOLL_CTL_MOD,
+                                               EPOLLIN | EPOLLPRI | EPOLLONESHOT);
+    }
 }
 
 bool epfd_info::get_and_unset_os_data_available()

src/core/iomux/epfd_info.h

@@ -135,6 +135,7 @@ class epfd_info : public lock_mutex_recursive, public cleanable_obj, public wake
     epoll_stats_t *m_stats;
     int m_log_invalid_events;
     bool m_b_os_data_available; // true when non offloaded data is available
+    bool use_os_events_in_internal_thread_epoll;
 
     int add_fd(int fd, epoll_event *event);
     int del_fd(int fd, bool passthrough = false);

src/core/main.cpp

@@ -904,6 +904,9 @@ void print_xlio_global_settings()
                     xlio_spec::to_str(MCE_SPEC_NVME_BF2));
     }
 #endif
+
+    VLOG_PARAM_NUMBER("OS events in epoll", safe_mce_sys().os_events_in_internal_thread_epoll,
+                      MCE_OS_EVENTS_IN_EPOLL, SYS_VAR_OS_EVENTS_IN_EPOLL);
 }
 
 void prepare_fork()

src/core/util/sys_vars.cpp

@@ -880,6 +880,7 @@ void mce_sys_var::get_env_params()
     handle_bf = MCE_DEFAULT_BF_FLAG;
     close_on_dup2 = MCE_DEFAULT_CLOSE_ON_DUP2;
     mtu = MCE_DEFAULT_MTU;
+    os_events_in_internal_thread_epoll = MCE_OS_EVENTS_IN_EPOLL;
 #if defined(DEFINED_NGINX)
     nginx_udp_socket_pool_size = MCE_DEFAULT_NGINX_UDP_POOL_SIZE;
     nginx_udp_socket_pool_rx_num_buffs_reuse = MCE_DEFAULT_NGINX_UDP_POOL_RX_NUM_BUFFS_REUSE;
@@ -960,6 +961,7 @@ void mce_sys_var::get_env_params()
             if (mce_spec == MCE_SPEC_NONE) {
                 mce_spec = MCE_SPEC_NGINX;
             }
+            os_events_in_internal_thread_epoll = false;
         }
     }
 #endif // DEFINED_NGINX
@@ -1967,6 +1969,10 @@ void mce_sys_var::get_env_params()
         mtu = (uint32_t)atoi(env_ptr);
     }
 
+    if ((env_ptr = getenv(SYS_VAR_OS_EVENTS_IN_EPOLL)) != NULL) {
+        os_events_in_internal_thread_epoll = atoi(env_ptr) ? true : false;
+    }
+
 #if defined(DEFINED_NGINX)
     if ((env_ptr = getenv(SYS_VAR_NGINX_UDP_POOL_SIZE)) != NULL) {
         nginx_udp_socket_pool_size = (uint32_t)atoi(env_ptr);

src/core/util/sys_vars.h

@@ -529,6 +529,7 @@ struct mce_sys_var {
             free_t memfree;
         } user_alloc;
     } m_ioctl;
+    bool os_events_in_internal_thread_epoll;
 
 private:
     void print_xlio_load_failure_msg();
@@ -669,6 +670,7 @@ extern mce_sys_var &safe_mce_sys();
 #define SYS_VAR_BF                        "XLIO_BF"
 #define SYS_VAR_CLOSE_ON_DUP2             "XLIO_CLOSE_ON_DUP2"
 #define SYS_VAR_MTU                       "XLIO_MTU"
+#define SYS_VAR_OS_EVENTS_IN_EPOLL        "XLIO_OS_EVENTS_IN_EPOLL"
 #if defined(DEFINED_NGINX)
 #define SYS_VAR_NGINX_WORKERS_NUM                 "XLIO_NGINX_WORKERS_NUM"
 #define SYS_VAR_NGINX_UDP_POOL_SIZE               "XLIO_NGINX_UDP_POOL_SIZE"
@@ -844,6 +846,7 @@ extern mce_sys_var &safe_mce_sys();
 #define MCE_DEFAULT_BF_FLAG                        (true)
 #define MCE_DEFAULT_CLOSE_ON_DUP2                  (true)
 #define MCE_DEFAULT_MTU                            (0)
+#define MCE_OS_EVENTS_IN_EPOLL                     (true)
 #if defined(DEFINED_NGINX)
 #define MCE_DEFAULT_NGINX_UDP_POOL_SIZE               (0)
 #define MCE_DEFAULT_NGINX_UDP_POOL_RX_NUM_BUFFS_REUSE (0)