livekit-plugins-browser: handle mouse/keyboard inputs on devmode

livekit-plugins/livekit-plugins-browser/cef/CMakeLists.txt

@@ -11,7 +11,8 @@ set(USE_SANDBOX OFF) # TODO(theomonnom): I don't think we want to enable sandbox
 
 # Specify the CEF distribution version.
 if(NOT DEFINED CEF_VERSION)
-  set(CEF_VERSION "122.1.10+gc902316+chromium-122.0.6261.112")
+  # set(CEF_VERSION "122.1.10+gc902316+chromium-122.0.6261.112")
+  set(CEF_VERSION "127.3.1+g6cbb30e+chromium-127.0.6533.100")
 endif()
 
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")

livekit-plugins/livekit-plugins-browser/cef/src/CMakeLists.txt

@@ -15,14 +15,17 @@ FetchContent_Declare(imgui GIT_REPOSITORY https://github.com/ocornut/imgui GIT_T
 FetchContent_GetProperties(imgui)
 FetchContent_MakeAvailable(imgui)
 file(GLOB IMGUI_SOURCES ${imgui_SOURCE_DIR}/*.cpp)
-file(GLOB IMGUI_HEADERS ${imgui_SOURCE_DIR}/*.h)
-add_library(imgui STATIC ${IMGUI_SOURCES} ${IMGUI_SOURCES} ${imgui_SOURCE_DIR}/backends/imgui_impl_glfw.cpp ${imgui_SOURCE_DIR}/backends/imgui_impl_opengl3.cpp)
+add_library(imgui STATIC ${IMGUI_SOURCES}
+        ${imgui_SOURCE_DIR}/backends/imgui_impl_glfw.cpp
+        ${imgui_SOURCE_DIR}/backends/imgui_impl_opengl3.cpp
+        ${imgui_SOURCE_DIR}/misc/cpp/imgui_stdlib.cpp
+)
 set_target_properties(imgui PROPERTIES CXX_STANDARD 17)
-target_include_directories(imgui PUBLIC ${imgui_SOURCE_DIR} ${imgui_SOURCE_DIR}/backends ${GLFW_INCLUDE_DIR})
+target_include_directories(imgui PUBLIC ${imgui_SOURCE_DIR} ${imgui_SOURCE_DIR}/misc/cpp ${imgui_SOURCE_DIR}/backends ${GLFW_INCLUDE_DIR})
 target_link_libraries(imgui PRIVATE glfw)
 
 
-set(LKCEF_SRCS app.cpp app.hpp handler.hpp handler.cpp dev_renderer.hpp dev_renderer.cpp)
+set(LKCEF_SRCS app.cpp app.hpp handler.hpp handler.cpp dev_renderer.hpp dev_renderer.cpp gleq.h)
 set(LKCEF_SRCS_LINUX main_linux.cpp)
 set(LKCEF_SRCS_MAC app_mac.mm)
 set(LKCEF_SRCS_WINDOWS main_win.cpp )

livekit-plugins/livekit-plugins-browser/cef/src/agents_python.cpp

@@ -16,7 +16,7 @@ std::shared_ptr<BrowserImpl> BrowserApp::CreateBrowser(
     const std::string& url,
     const BrowserOptions& options) {
 
-  app_->CreateBrowser(url, options.framerate, options.created_callback);
+  app_->CreateBrowser(url, options.framerate, options.width, options.height, options.created_callback);
   return nullptr;//std::make_shared<BrowserImpl>();
 }
 
@@ -40,6 +40,8 @@ PYBIND11_MODULE(lkcef_python, m) {
   py::class_<BrowserOptions>(m, "BrowserOptions")
       .def(py::init())
       .def_readwrite("framerate", &BrowserOptions::framerate)
+      .def_readwrite("width", &BrowserOptions::width)
+      .def_readwrite("height", &BrowserOptions::height)
       .def_readwrite("created_callback", &BrowserOptions::created_callback);
 
   py::class_<BrowserApp>(m, "BrowserApp")

livekit-plugins/livekit-plugins-browser/cef/src/agents_python.hpp

@@ -15,6 +15,8 @@ struct AppOptions {
 
 struct BrowserOptions {
   int framerate = 30;
+  int width = 800;
+  int height = 600;
   std::function<void()> created_callback = nullptr;
 };
 

livekit-plugins/livekit-plugins-browser/cef/src/app.cpp

@@ -38,6 +38,8 @@ CefRefPtr<CefClient> AgentApp::GetDefaultClient() {
 CefRefPtr<BrowserHandle> AgentApp::CreateBrowser(
     const std::string& url,
     int framerate,
+    int width,
+    int height,
     std::function<void()> created_callback) {
   CEF_REQUIRE_UI_THREAD();
   CefWindowInfo windowInfo;
@@ -51,7 +53,7 @@ CefRefPtr<BrowserHandle> AgentApp::CreateBrowser(
   settings.background_color = CefColorSetARGB(255, 255, 255, 255);
 
   CefRefPtr<BrowserHandle> browser_handle =
-        new BrowserHandle(created_callback);
+        new BrowserHandle(created_callback, width, height);
 
   client_->AddPendingHandle(browser_handle);
 

livekit-plugins/livekit-plugins-browser/cef/src/app.hpp

@@ -28,10 +28,14 @@ class AgentApp : public CefApp, public CefBrowserProcessHandler {
   CefRefPtr<BrowserHandle> CreateBrowser(
       const std::string& url,
       int framerate,
+      int width,
+      int height,
       std::function<void()> created_callback);
 
   int Run();
 
+  bool IsDevMode() const { return dev_mode_; }
+
  private:
   IMPLEMENT_REFCOUNTING(AgentApp);
 

livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm

@@ -3,87 +3,74 @@
 
 #include <iostream>
 
+#import <Cocoa/Cocoa.h>
+#include <objc/runtime.h>
+
 #include "app.hpp"
 #include "handler.hpp"
 #include "include/cef_application_mac.h"
 #include "include/cef_command_line.h"
 #include "include/wrapper/cef_library_loader.h"
 
-// Receives notifications from the application.
-@interface AgentsAppDelegate : NSObject <NSApplicationDelegate>
+BOOL g_handling_send_event = false;
+
+@interface NSApplication (AgentsApplication)
+
+- (BOOL)isHandlingSendEvent;
+- (void)setHandlingSendEvent:(BOOL)handlingSendEvent;
+- (void)_swizzled_sendEvent:(NSEvent*)event;
+- (void)_swizzled_terminate:(id)sender;
 
-- (void)createApplication:(id)object;
-- (void)tryToTerminateApplication:(NSApplication*)app;
 @end
 
-// Provide the CefAppProtocol implementation required by CEF.
-@interface AgentsApplication : NSApplication <CefAppProtocol> {
- @private
-  BOOL handlingSendEvent_;
+@implementation NSApplication (AgentsApplication)
+
+// This selector is called very early during the application initialization.
++ (void)load {
+  NSLog(@"AgentsApplication::load");
+  // Swap NSApplication::sendEvent with _swizzled_sendEvent.
+  Method original = class_getInstanceMethod(self, @selector(sendEvent));
+  Method swizzled =
+      class_getInstanceMethod(self, @selector(_swizzled_sendEvent));
+  method_exchangeImplementations(original, swizzled);
+
+  Method originalTerm = class_getInstanceMethod(self, @selector(terminate:));
+  Method swizzledTerm =
+      class_getInstanceMethod(self, @selector(_swizzled_terminate:));
+  method_exchangeImplementations(originalTerm, swizzledTerm);
 }
-@end
 
-@implementation AgentsApplication
 - (BOOL)isHandlingSendEvent {
-  return handlingSendEvent_;
+  return g_handling_send_event;
 }
 
 - (void)setHandlingSendEvent:(BOOL)handlingSendEvent {
-  handlingSendEvent_ = handlingSendEvent;
+  g_handling_send_event = handlingSendEvent;
 }
 
-- (void)sendEvent:(NSEvent*)event {
+- (void)_swizzled_sendEvent:(NSEvent*)event {
   CefScopedSendingEvent sendingEventScoper;
-  [super sendEvent:event];
-}
-
-- (void)terminate:(id)sender {
-  AgentsAppDelegate* delegate =
-      static_cast<AgentsAppDelegate*>([NSApp delegate]);
-  [delegate tryToTerminateApplication:self];
-  // Return, don't exit. The application is responsible for exiting on its own.
-}
-@end
-
-@implementation AgentsAppDelegate
-
-// Create the application on the UI thread.
-- (void)createApplication:(id)object {
-  [[NSBundle mainBundle] loadNibNamed:@"MainMenu"
-                                owner:NSApp
-                      topLevelObjects:nil];
-
-  // Set the delegate for application events.
-  [[NSApplication sharedApplication] setDelegate:self];
-}
-
-- (void)tryToTerminateApplication:(NSApplication*)app {
+  // Calls NSApplication::sendEvent due to the swizzling.
+  [self _swizzled_sendEvent:event];
 }
 
-- (NSApplicationTerminateReply)applicationShouldTerminate:
-    (NSApplication*)sender {
-  return NSTerminateNow;
+- (void)_swizzled_terminate:(id)sender {
+  [self _swizzled_terminate:sender];
 }
 
-// Called when the user clicks the app dock icon while the application is
-// already running.
-- (BOOL)applicationShouldHandleReopen:(NSApplication*)theApplication
-                    hasVisibleWindows:(BOOL)flag {
-  return NO;
-}
 @end
 
 // Entry point function for the browser process.
 int RunAgentApp(CefRefPtr<AgentApp> app) {
   CefMainArgs main_args(0, nullptr);
 
   @autoreleasepool {
-    [AgentsApplication sharedApplication];
+    //[NSApplication sharedApplication];
 
     // If there was an invocation to NSApp prior to this method, then the NSApp
     // will not be a AgentsApplication, but will instead be an NSApplication.
     // This is undesirable and we must enforce that this doesn't happen.
-    CHECK([NSApp isKindOfClass:[AgentsApplication class]]);
+    //CHECK([NSApp isKindOfClass:[NSApplication class]]);
 
     std::string framework_path =
         "/Users/theomonnom/livekit/agents/livekit-plugins/"
@@ -104,6 +91,8 @@ int RunAgentApp(CefRefPtr<AgentApp> app) {
     }
 
     CefSettings settings{};
+    settings.chrome_runtime = true;
+    settings.external_message_pump = app->IsDevMode();
     // settings.remote_debugging_port = 8088;
     CefString(&settings.framework_dir_path).FromString(framework_path);
     CefString(&settings.main_bundle_path).FromString(main_bundle_path);
@@ -122,24 +111,12 @@ int RunAgentApp(CefRefPtr<AgentApp> app) {
       return 1;
     }
 
-    // Create the application delegate.
-    AgentsAppDelegate* delegate = [[AgentsAppDelegate alloc] init];
-    // Set as the delegate for application events.
-    NSApp.delegate = delegate;
-
-    [delegate performSelectorOnMainThread:@selector(createApplication:)
-                               withObject:nil
-                            waitUntilDone:NO];
 
     app->Run();
 
     CefShutdown();
-    cef_unload_library();
 
-#if !__has_feature(objc_arc)
-    [delegate release];
-#endif  // !__has_feature(objc_arc)
-    delegate = nil;
+    cef_unload_library();
   }  // @autoreleasepool
 
   return 0;