From 40d33a9b68d83faca28c55e25ca34e7122267d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?The=CC=81o=20Monnom?= Date: Sun, 18 Aug 2024 11:20:45 -0700 Subject: [PATCH 1/6] wip --- .../cef/src/CMakeLists.txt | 11 +- .../cef/src/agents_python.cpp | 4 +- .../cef/src/agents_python.hpp | 2 + .../livekit-plugins-browser/cef/src/app.cpp | 4 +- .../livekit-plugins-browser/cef/src/app.hpp | 2 + .../cef/src/dev_renderer.cpp | 245 ++++++++-- .../cef/src/dev_renderer.hpp | 16 +- .../livekit-plugins-browser/cef/src/gleq.h | 419 ++++++++++++++++++ .../cef/src/handler.cpp | 38 +- .../cef/src/handler.hpp | 85 +++- 10 files changed, 746 insertions(+), 80 deletions(-) create mode 100644 livekit-plugins/livekit-plugins-browser/cef/src/gleq.h diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/CMakeLists.txt b/livekit-plugins/livekit-plugins-browser/cef/src/CMakeLists.txt index 648f16864..d7be7bb6e 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/CMakeLists.txt +++ b/livekit-plugins/livekit-plugins-browser/cef/src/CMakeLists.txt @@ -15,14 +15,17 @@ FetchContent_Declare(imgui GIT_REPOSITORY https://github.com/ocornut/imgui GIT_T FetchContent_GetProperties(imgui) FetchContent_MakeAvailable(imgui) file(GLOB IMGUI_SOURCES ${imgui_SOURCE_DIR}/*.cpp) -file(GLOB IMGUI_HEADERS ${imgui_SOURCE_DIR}/*.h) -add_library(imgui STATIC ${IMGUI_SOURCES} ${IMGUI_SOURCES} ${imgui_SOURCE_DIR}/backends/imgui_impl_glfw.cpp ${imgui_SOURCE_DIR}/backends/imgui_impl_opengl3.cpp) +add_library(imgui STATIC ${IMGUI_SOURCES} + ${imgui_SOURCE_DIR}/backends/imgui_impl_glfw.cpp + ${imgui_SOURCE_DIR}/backends/imgui_impl_opengl3.cpp + ${imgui_SOURCE_DIR}/misc/cpp/imgui_stdlib.cpp +) set_target_properties(imgui PROPERTIES CXX_STANDARD 17) -target_include_directories(imgui PUBLIC ${imgui_SOURCE_DIR} ${imgui_SOURCE_DIR}/backends ${GLFW_INCLUDE_DIR}) +target_include_directories(imgui PUBLIC ${imgui_SOURCE_DIR} ${imgui_SOURCE_DIR}/misc/cpp ${imgui_SOURCE_DIR}/backends ${GLFW_INCLUDE_DIR}) target_link_libraries(imgui PRIVATE glfw) -set(LKCEF_SRCS app.cpp app.hpp handler.hpp handler.cpp dev_renderer.hpp dev_renderer.cpp) +set(LKCEF_SRCS app.cpp app.hpp handler.hpp handler.cpp dev_renderer.hpp dev_renderer.cpp gleq.h) set(LKCEF_SRCS_LINUX main_linux.cpp) set(LKCEF_SRCS_MAC app_mac.mm) set(LKCEF_SRCS_WINDOWS main_win.cpp ) diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/agents_python.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/agents_python.cpp index 7e44d624a..67be7ee7e 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/agents_python.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/agents_python.cpp @@ -16,7 +16,7 @@ std::shared_ptr BrowserApp::CreateBrowser( const std::string& url, const BrowserOptions& options) { - app_->CreateBrowser(url, options.framerate, options.created_callback); + app_->CreateBrowser(url, options.framerate, options.width, options.height, options.created_callback); return nullptr;//std::make_shared(); } @@ -40,6 +40,8 @@ PYBIND11_MODULE(lkcef_python, m) { py::class_(m, "BrowserOptions") .def(py::init()) .def_readwrite("framerate", &BrowserOptions::framerate) + .def_readwrite("width", &BrowserOptions::width) + .def_readwrite("height", &BrowserOptions::height) .def_readwrite("created_callback", &BrowserOptions::created_callback); py::class_(m, "BrowserApp") diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/agents_python.hpp b/livekit-plugins/livekit-plugins-browser/cef/src/agents_python.hpp index e77a59776..8069045f8 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/agents_python.hpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/agents_python.hpp @@ -15,6 +15,8 @@ struct AppOptions { struct BrowserOptions { int framerate = 30; + int width = 800; + int height = 600; std::function created_callback = nullptr; }; diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/app.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/app.cpp index 1dfbe7976..1f15635c7 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/app.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/app.cpp @@ -38,6 +38,8 @@ CefRefPtr AgentApp::GetDefaultClient() { CefRefPtr AgentApp::CreateBrowser( const std::string& url, int framerate, + int width, + int height, std::function created_callback) { CEF_REQUIRE_UI_THREAD(); CefWindowInfo windowInfo; @@ -51,7 +53,7 @@ CefRefPtr AgentApp::CreateBrowser( settings.background_color = CefColorSetARGB(255, 255, 255, 255); CefRefPtr browser_handle = - new BrowserHandle(created_callback); + new BrowserHandle(created_callback, width, height); client_->AddPendingHandle(browser_handle); diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/app.hpp b/livekit-plugins/livekit-plugins-browser/cef/src/app.hpp index aa5b8d1ab..b3c812173 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/app.hpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/app.hpp @@ -28,6 +28,8 @@ class AgentApp : public CefApp, public CefBrowserProcessHandler { CefRefPtr CreateBrowser( const std::string& url, int framerate, + int width, + int height, std::function created_callback); int Run(); diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp index a1e10d316..52dd9cf32 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp @@ -2,13 +2,19 @@ #include +#include "handler.hpp" + +#define IMGUI_DEFINE_MATH_OPERATORS #include "imgui.h" #include "imgui_impl_glfw.h" #include "imgui_impl_opengl3.h" - +#include "imgui_stdlib.h" +#include "include/cef_app.h" #include "include/wrapper/cef_helpers.h" -#include "include/cef_app.h" +#define GLEQ_IMPLEMENTATION +#define GLEQ_STATIC +#include "gleq.h" // DCHECK on gl errors. #if DCHECK_IS_ON() @@ -25,7 +31,111 @@ static void glfw_error_callback(int error, const char* description) { fprintf(stderr, "GLFW Error %d: %s\n", error, description); } -DevRenderer::DevRenderer() { +/*static void glfw_key_callback(GLFWwindow* window, int key, int scancode, int +action, int mods) { DevRenderer* dev_renderer = +static_cast(glfwGetWindowUserPointer(window)); + + if (action == GLFW_PRESS || action == GLFW_RELEASE) { + // Handle key event forwarding to CEF + CefKeyEvent cef_event; + cef_event.windows_key_code = key; + cef_event.native_key_code = scancode; + cef_event.is_system_key = false; + cef_event.type = (action == GLFW_PRESS) ? KEYEVENT_RAWKEYDOWN : +KEYEVENT_KEYUP; cef_event.modifiers = mods; + + auto browser = GetBrowserHandleFromWindow(window); + browser->GetHost()->SendKeyEvent(cef_event); + } +} + + + +void glfw_char_callback(GLFWwindow* window, unsigned int codepoint) { + DevRenderer* dev_renderer = +static_cast(glfwGetWindowUserPointer(window)); + + CefKeyEvent cef_event; + cef_event.character = codepoint; + cef_event.type = KEYEVENT_CHAR; + + auto browser = GetBrowserHandleFromWindow(window); + browser->GetHost()->SendKeyEvent(cef_event); +}*/ + +/*static void glfw_mouse_button_callback(GLFWwindow* window, int button, int +action, int mods) { DevRenderer* dev_renderer = +static_cast(glfwGetWindowUserPointer(window)); + + if (button >= 0 && button <= 2) { // GLFW only supports buttons 0-2 (left, +right, middle) CefMouseEvent cef_event; double xpos, ypos; + glfwGetCursorPos(window, &xpos, &ypos); + cef_event.x = static_cast(xpos); + cef_event.y = static_cast(ypos); + + auto browser = GetBrowserHandleFromWindow(window); + + if (action == GLFW_PRESS) { + browser->GetHost()->SendMouseClickEvent(cef_event, +static_cast(button), false, 1); } else if (action == +GLFW_RELEASE) { browser->GetHost()->SendMouseClickEvent(cef_event, +static_cast(button), true, 1); + } + } +} + +// Mouse move callback +static void glfw_cursor_position_callback(GLFWwindow* window, double xpos, +double ypos) { DevRenderer* dev_renderer = +static_cast(glfwGetWindowUserPointer(window)); + DevRenderer::BrowserData* data = dev_renderer->getSelectedBrowserData(); + + if (!data) return; + + CefMouseEvent cef_event; + cef_event.x = static_cast(xpos); + cef_event.y = static_cast(ypos); + + // Translate the coordinate to the ImGUI window + + data->browser->GetHost()->SendMouseMoveEvent(cef_event, false); +} + +// Scroll callback +static void glfw_scroll_callback(GLFWwindow* window, double xoffset, double +yoffset) { DevRenderer* dev_renderer = +static_cast(glfwGetWindowUserPointer(window)); + + CefMouseEvent cef_event; + double xpos, ypos; + glfwGetCursorPos(window, &xpos, &ypos); + cef_event.x = static_cast(xpos); + cef_event.y = static_cast(ypos); + + auto browser = GetBrowserHandleFromWindow(window); + browser->GetHost()->SendMouseWheelEvent(cef_event, static_cast(xoffset * +100), static_cast(yoffset * 100)); +}*/ + +DevRenderer::DevRenderer() {} + +void DevRenderer::OnTitleChange(CefRefPtr browser, + const CefString& title) { + CEF_REQUIRE_UI_THREAD(); + int identifier = browser->GetIdentifier(); + BrowserData* data = &browser_data_[identifier]; + data->title = title; +} + +void DevRenderer::OnLoadingStateChange(CefRefPtr browser, + bool isLoading, + bool canGoBack, + bool canGoForward) { + if (!isLoading) { + int identifier = browser->GetIdentifier(); + BrowserData* data = &browser_data_[identifier]; + data->url = browser->GetMainFrame()->GetURL(); + } } void DevRenderer::OnAfterCreated(CefRefPtr browser) { @@ -36,9 +146,10 @@ void DevRenderer::OnAfterCreated(CefRefPtr browser) { glGenTextures(1, &texture_id); VERIFY_NO_ERROR; - RenderData render_data{}; - render_data.texture_id = texture_id; - render_data_.insert({identifier, render_data}); + BrowserData data{}; + data.browser = browser; + data.texture_id = texture_id; + browser_data_.insert({identifier, data}); glBindTexture(GL_TEXTURE_2D, texture_id); VERIFY_NO_ERROR; @@ -48,42 +159,42 @@ void DevRenderer::OnAfterCreated(CefRefPtr browser) { } void DevRenderer::OnPaint(CefRefPtr browser, - CefRenderHandler::PaintElementType type, - const CefRenderHandler::RectList& dirtyRects, - const void* buffer, - int width, - int height) { + CefRenderHandler::PaintElementType type, + const CefRenderHandler::RectList& dirtyRects, + const void* buffer, + int width, + int height) { CEF_REQUIRE_UI_THREAD(); - if (type != CefRenderHandler::PaintElementType::PET_VIEW){ + if (type != CefRenderHandler::PaintElementType::PET_VIEW) { std::cout << "Ignoring PET_POPUP" << std::endl; - return; // Ignore PET_POPUP for now, bc I'm lazy + return; // Ignore PET_POPUP for now, bc I'm lazy } int identifier = browser->GetIdentifier(); - RenderData* render_data = &render_data_[identifier]; + BrowserData* data = &browser_data_[identifier]; - int old_width = render_data->view_width; - int old_height = render_data->view_height; + int old_width = data->view_width; + int old_height = data->view_height; - render_data->view_width = width; - render_data->view_height = height; + data->view_width = width; + data->view_height = height; - glBindTexture(GL_TEXTURE_2D, render_data->texture_id); + glBindTexture(GL_TEXTURE_2D, data->texture_id); glPixelStorei(GL_UNPACK_ROW_LENGTH, width); VERIFY_NO_ERROR; - bool has_fullscreen_rect = dirtyRects.size() == 1 && - dirtyRects[0] == CefRect(0, 0, width, height); + bool has_fullscreen_rect = + dirtyRects.size() == 1 && dirtyRects[0] == CefRect(0, 0, width, height); if (old_width != width || old_height != height || has_fullscreen_rect) { glPixelStorei(GL_UNPACK_SKIP_PIXELS, 0); VERIFY_NO_ERROR; glPixelStorei(GL_UNPACK_SKIP_ROWS, 0); VERIFY_NO_ERROR; - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, - GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, buffer); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, GL_BGRA, + GL_UNSIGNED_INT_8_8_8_8_REV, buffer); VERIFY_NO_ERROR; } else { CefRenderHandler::RectList::const_iterator i = dirtyRects.begin(); @@ -93,9 +204,8 @@ void DevRenderer::OnPaint(CefRefPtr browser, VERIFY_NO_ERROR; glPixelStorei(GL_UNPACK_SKIP_ROWS, rect.y); VERIFY_NO_ERROR; - glTexSubImage2D(GL_TEXTURE_2D, 0, rect.x, rect.y, rect.width, - rect.height, GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, - buffer); + glTexSubImage2D(GL_TEXTURE_2D, 0, rect.x, rect.y, rect.width, rect.height, + GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, buffer); VERIFY_NO_ERROR; } } @@ -104,9 +214,9 @@ void DevRenderer::OnPaint(CefRefPtr browser, void DevRenderer::OnBeforeClose(CefRefPtr browser) { CEF_REQUIRE_UI_THREAD(); int identifier = browser->GetIdentifier(); - RenderData* render_data = &render_data_[identifier]; - glDeleteTextures(1, &render_data->texture_id); - render_data_.erase(identifier); + BrowserData* data = &browser_data_[identifier]; + glDeleteTextures(1, &data->texture_id); + browser_data_.erase(identifier); } void DevRenderer::Run() { @@ -117,6 +227,8 @@ void DevRenderer::Run() { return; } + gleqInit(); + const char* glsl_version = "#version 150"; glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3); glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 2); @@ -127,6 +239,8 @@ void DevRenderer::Run() { glfwCreateWindow(800, 600, "livekit-plugins-browser (Development Window)", nullptr, nullptr); + gleqTrackWindow(window_); + if (!window_) { std::cerr << "Failed to create GLFW window" << std::endl; glfwTerminate(); @@ -142,11 +256,9 @@ void DevRenderer::Run() { io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard; io.ConfigFlags |= ImGuiConfigFlags_DockingEnable; - // Setup Platform/Renderer backends ImGui_ImplGlfw_InitForOpenGL(window_, true); ImGui_ImplOpenGL3_Init(glsl_version); - ImVec4 clear_color = ImVec4(0.45f, 0.55f, 0.60f, 1.00f); while (!glfwWindowShouldClose(window_)) { glfwPollEvents(); @@ -156,20 +268,73 @@ void DevRenderer::Run() { ImGui_ImplOpenGL3_NewFrame(); ImGui_ImplGlfw_NewFrame(); ImGui::NewFrame(); - ImGui::ShowDemoWindow(); - - for (auto& [identifier, render_data] : render_data_) { - ImGui::Begin("Browser"); - ImGui::Text("Browser %d", identifier); - ImGui::Image((void*)(intptr_t)render_data.texture_id, - ImVec2(render_data.view_width, render_data.view_height)); + // Focused browser input states + BrowserData* focused_browser = nullptr; + int browser_view_x = 0; + int browser_view_y = 0; + + for (auto& [identifier, data] : browser_data_) { + std::string name = + (data.title.empty() ? "Browser #" + std::to_string(identifier) + : data.title) + + "###Browser" + std::to_string(identifier); + + if (ImGui::Begin(name.c_str())) { + ImVec2 size = ImGui::GetContentRegionAvail(); + + // Resize the browser view if needed + if (size.x > 0 && size.y > 0 && + (data.view_width != static_cast(size.x) || + data.view_height != static_cast(size.y))) { + AgentHandler::GetInstance() + ->GetBrowserHandle(identifier) + ->SetSize(static_cast(size.x), static_cast(size.y)); + } + + if (ImGui::InputText("URL", &data.url, + ImGuiInputTextFlags_EnterReturnsTrue)) { + data.browser->GetMainFrame()->LoadURL(data.url); + } + + ImVec2 cursor_pos = ImGui::GetCursorScreenPos(); + + bool is_focused = ImGui::IsWindowFocused(); + if (is_focused) { + focused_browser = &data; + browser_view_x = static_cast(cursor_pos.x); + browser_view_y = static_cast(cursor_pos.y); + } + + // Render the browser tex + ImGui::Image((void*)(intptr_t)data.texture_id, + ImVec2((float)data.view_width, (float)data.view_height)); + } ImGui::End(); } + GLEQevent event; + + while (gleqNextEvent(&event)) { + switch (event.type) { + case GLEQ_CURSOR_MOVED: + + if (focused_browser) { + CefMouseEvent cef_event; + cef_event.x = event.pos.x - browser_view_x; + cef_event.y = event.pos.y - browser_view_y; + focused_browser->browser->GetHost()->SendMouseMoveEvent(cef_event, + false); + } + break; + default: + break; + } + + gleqFreeEvent(&event); + } - // Rendering ImGui::Render(); int display_w, display_h; glfwGetFramebufferSize(window_, &display_w, &display_h); @@ -191,5 +356,5 @@ void DevRenderer::Run() { } void DevRenderer::Close() { - //glfwSetWindowShouldClose(window_, GLFW_TRUE); + // glfwSetWindowShouldClose(window_, GLFW_TRUE); } diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.hpp b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.hpp index 673674474..4799e86a0 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.hpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.hpp @@ -18,6 +18,13 @@ class DevRenderer: public CefBaseRefCounted { void Run(); void Close(); + void OnTitleChange(CefRefPtr browser, + const CefString &title); + + void OnLoadingStateChange(CefRefPtr browser, + bool isLoading, + bool canGoBack, + bool canGoForward); void OnAfterCreated(CefRefPtr browser); @@ -30,19 +37,22 @@ class DevRenderer: public CefBaseRefCounted { void OnBeforeClose(CefRefPtr browser); - void* getNativeWindowHandle() { + void* getNativeWindowHandle() const { return glfwGetCocoaWindow(window_); } private: - struct RenderData{ + struct BrowserData{ + CefRefPtr browser; unsigned int texture_id; int view_width; int view_height; + std::string title; + std::string url; }; GLFWwindow* window_ = nullptr; - std::unordered_map render_data_; + std::unordered_map browser_data_; IMPLEMENT_REFCOUNTING(DevRenderer); }; diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/gleq.h b/livekit-plugins/livekit-plugins-browser/cef/src/gleq.h new file mode 100644 index 000000000..69a9e6293 --- /dev/null +++ b/livekit-plugins/livekit-plugins-browser/cef/src/gleq.h @@ -0,0 +1,419 @@ +/* +* GLEQ - A basic event queue for GLFW 3 +* Copyright © Camilla Löwy +* +* This software is provided 'as-is', without any express or implied +* warranty. In no event will the authors be held liable for any damages +* arising from the use of this software. +* +* Permission is granted to anyone to use this software for any purpose, +* including commercial applications, and to alter it and redistribute it +* freely, subject to the following restrictions: +* +* 1. The origin of this software must not be misrepresented; you must not +* claim that you wrote the original software. If you use this software +* in a product, an acknowledgment in the product documentation would +* be appreciated but is not required. +* +* 2. Altered source versions must be plainly marked as such, and must not +* be misrepresented as being the original software. +* +* 3. This notice may not be removed or altered from any source +* distribution. +*/ + +#ifndef GLEQ_HEADER_FILE +#define GLEQ_HEADER_FILE + +#include + +#ifdef GLEQ_STATIC +#define GLEQDEF static +#else +#define GLEQDEF extern +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum +{ + GLEQ_NONE, + GLEQ_WINDOW_MOVED, + GLEQ_WINDOW_RESIZED, + GLEQ_WINDOW_CLOSED, + GLEQ_WINDOW_REFRESH, + GLEQ_WINDOW_FOCUSED, + GLEQ_WINDOW_DEFOCUSED, + GLEQ_WINDOW_ICONIFIED, + GLEQ_WINDOW_UNICONIFIED, + GLEQ_FRAMEBUFFER_RESIZED, + GLEQ_BUTTON_PRESSED, + GLEQ_BUTTON_RELEASED, + GLEQ_CURSOR_MOVED, + GLEQ_CURSOR_ENTERED, + GLEQ_CURSOR_LEFT, + GLEQ_SCROLLED, + GLEQ_KEY_PRESSED, + GLEQ_KEY_REPEATED, + GLEQ_KEY_RELEASED, + GLEQ_CODEPOINT_INPUT, + GLEQ_MONITOR_CONNECTED, + GLEQ_MONITOR_DISCONNECTED, +#if GLFW_VERSION_MINOR >= 1 + GLEQ_FILE_DROPPED, +#endif +#if GLFW_VERSION_MINOR >= 2 + GLEQ_JOYSTICK_CONNECTED, + GLEQ_JOYSTICK_DISCONNECTED, +#endif +#if GLFW_VERSION_MINOR >= 3 + GLEQ_WINDOW_MAXIMIZED, + GLEQ_WINDOW_UNMAXIMIZED, + GLEQ_WINDOW_SCALE_CHANGED, +#endif +} GLEQtype; + +typedef struct GLEQevent +{ + GLEQtype type; + union { + GLFWwindow* window; + GLFWmonitor* monitor; + int joystick; + }; + union { + struct { + int x; + int y; + } pos; + struct { + int width; + int height; + } size; + struct { + double x; + double y; + } scroll; + struct { + int key; + int scancode; + int mods; + } keyboard; + struct { + int button; + int mods; + } mouse; + unsigned int codepoint; +#if GLFW_VERSION_MINOR >= 1 + struct { + char** paths; + int count; + } file; +#endif +#if GLFW_VERSION_MINOR >= 3 + struct { + float x; + float y; + } scale; +#endif + }; +} GLEQevent; + +GLEQDEF void gleqInit(void); +GLEQDEF void gleqTrackWindow(GLFWwindow* window); + +GLEQDEF int gleqNextEvent(GLEQevent* event); +GLEQDEF void gleqFreeEvent(GLEQevent* event); + +#ifdef __cplusplus +} +#endif + +#ifdef GLEQ_IMPLEMENTATION + +#include +#include +#include + +#ifndef GLEQ_CAPACITY +#define GLEQ_CAPACITY 1024 +#endif + +static struct +{ + GLEQevent events[GLEQ_CAPACITY]; + size_t head; + size_t tail; +} gleq_queue = { {}, 0, 0 }; + +static char* gleq_strdup(const char* string) +{ + const size_t size = strlen(string) + 1; + char* result = (char*) malloc(size); + memcpy(result, string, size); + return result; +} + +static GLEQevent* gleq_new_event(void) +{ + GLEQevent* event = gleq_queue.events + gleq_queue.head; + gleq_queue.head = (gleq_queue.head + 1) % GLEQ_CAPACITY; + assert(gleq_queue.head != gleq_queue.tail); + memset(event, 0, sizeof(GLEQevent)); + return event; +} + +static void gleq_window_pos_callback(GLFWwindow* window, int x, int y) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_WINDOW_MOVED; + event->window = window; + event->pos.x = x; + event->pos.y = y; +} + +static void gleq_window_size_callback(GLFWwindow* window, int width, int height) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_WINDOW_RESIZED; + event->window = window; + event->size.width = width; + event->size.height = height; +} + +static void gleq_window_close_callback(GLFWwindow* window) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_WINDOW_CLOSED; + event->window = window; +} + +static void gleq_window_refresh_callback(GLFWwindow* window) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_WINDOW_REFRESH; + event->window = window; +} + +static void gleq_window_focus_callback(GLFWwindow* window, int focused) +{ + GLEQevent* event = gleq_new_event(); + event->window = window; + + if (focused) + event->type = GLEQ_WINDOW_FOCUSED; + else + event->type = GLEQ_WINDOW_DEFOCUSED; +} + +static void gleq_window_iconify_callback(GLFWwindow* window, int iconified) +{ + GLEQevent* event = gleq_new_event(); + event->window = window; + + if (iconified) + event->type = GLEQ_WINDOW_ICONIFIED; + else + event->type = GLEQ_WINDOW_UNICONIFIED; +} + +static void gleq_framebuffer_size_callback(GLFWwindow* window, int width, int height) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_FRAMEBUFFER_RESIZED; + event->window = window; + event->size.width = width; + event->size.height = height; +} + +static void gleq_mouse_button_callback(GLFWwindow* window, int button, int action, int mods) +{ + GLEQevent* event = gleq_new_event(); + event->window = window; + event->mouse.button = button; + event->mouse.mods = mods; + + if (action == GLFW_PRESS) + event->type = GLEQ_BUTTON_PRESSED; + else if (action == GLFW_RELEASE) + event->type = GLEQ_BUTTON_RELEASED; +} + +static void gleq_cursor_pos_callback(GLFWwindow* window, double x, double y) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_CURSOR_MOVED; + event->window = window; + event->pos.x = (int) x; + event->pos.y = (int) y; +} + +static void gleq_cursor_enter_callback(GLFWwindow* window, int entered) +{ + GLEQevent* event = gleq_new_event(); + event->window = window; + + if (entered) + event->type = GLEQ_CURSOR_ENTERED; + else + event->type = GLEQ_CURSOR_LEFT; +} + +static void gleq_scroll_callback(GLFWwindow* window, double x, double y) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_SCROLLED; + event->window = window; + event->scroll.x = x; + event->scroll.y = y; +} + +static void gleq_key_callback(GLFWwindow* window, int key, int scancode, int action, int mods) +{ + GLEQevent* event = gleq_new_event(); + event->window = window; + event->keyboard.key = key; + event->keyboard.scancode = scancode; + event->keyboard.mods = mods; + + if (action == GLFW_PRESS) + event->type = GLEQ_KEY_PRESSED; + else if (action == GLFW_RELEASE) + event->type = GLEQ_KEY_RELEASED; + else if (action == GLFW_REPEAT) + event->type = GLEQ_KEY_REPEATED; +} + +static void gleq_char_callback(GLFWwindow* window, unsigned int codepoint) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_CODEPOINT_INPUT; + event->window = window; + event->codepoint = codepoint; +} + +static void gleq_monitor_callback(GLFWmonitor* monitor, int action) +{ + GLEQevent* event = gleq_new_event(); + event->monitor = monitor; + + if (action == GLFW_CONNECTED) + event->type = GLEQ_MONITOR_CONNECTED; + else if (action == GLFW_DISCONNECTED) + event->type = GLEQ_MONITOR_DISCONNECTED; +} + +#if GLFW_VERSION_MINOR >= 1 +static void gleq_file_drop_callback(GLFWwindow* window, int count, const char** paths) +{ + GLEQevent* event = gleq_new_event(); + event->type = GLEQ_FILE_DROPPED; + event->window = window; + event->file.paths = (char**) malloc(count * sizeof(char*)); + event->file.count = count; + + while (count--) + event->file.paths[count] = gleq_strdup(paths[count]); +} +#endif + +#if GLFW_VERSION_MINOR >= 2 +static void gleq_joystick_callback(int jid, int action) +{ + GLEQevent* event = gleq_new_event(); + event->joystick = jid; + + if (action == GLFW_CONNECTED) + event->type = GLEQ_JOYSTICK_CONNECTED; + else if (action == GLFW_DISCONNECTED) + event->type = GLEQ_JOYSTICK_DISCONNECTED; +} +#endif + +#if GLFW_VERSION_MINOR >= 3 +static void gleq_window_maximize_callback(GLFWwindow* window, int maximized) +{ + GLEQevent* event = gleq_new_event(); + event->window = window; + + if (maximized) + event->type = GLEQ_WINDOW_MAXIMIZED; + else + event->type = GLEQ_WINDOW_UNMAXIMIZED; +} + +static void gleq_window_content_scale_callback(GLFWwindow* window, float xscale, float yscale) +{ + GLEQevent* event = gleq_new_event(); + event->window = window; + event->type = GLEQ_WINDOW_SCALE_CHANGED; + event->scale.x = xscale; + event->scale.y = yscale; +} +#endif + +GLEQDEF void gleqInit(void) +{ + glfwSetMonitorCallback(gleq_monitor_callback); +#if GLFW_VERSION_MINOR >= 2 + glfwSetJoystickCallback(gleq_joystick_callback); +#endif +} + +GLEQDEF void gleqTrackWindow(GLFWwindow* window) +{ + glfwSetWindowPosCallback(window, gleq_window_pos_callback); + glfwSetWindowSizeCallback(window, gleq_window_size_callback); + glfwSetWindowCloseCallback(window, gleq_window_close_callback); + glfwSetWindowRefreshCallback(window, gleq_window_refresh_callback); + glfwSetWindowFocusCallback(window, gleq_window_focus_callback); + glfwSetWindowIconifyCallback(window, gleq_window_iconify_callback); + glfwSetFramebufferSizeCallback(window, gleq_framebuffer_size_callback); + glfwSetMouseButtonCallback(window, gleq_mouse_button_callback); + glfwSetCursorPosCallback(window, gleq_cursor_pos_callback); + glfwSetCursorEnterCallback(window, gleq_cursor_enter_callback); + glfwSetScrollCallback(window, gleq_scroll_callback); + glfwSetKeyCallback(window, gleq_key_callback); + glfwSetCharCallback(window, gleq_char_callback); +#if GLFW_VERSION_MINOR >= 1 + glfwSetDropCallback(window, gleq_file_drop_callback); +#endif +#if GLFW_VERSION_MINOR >= 3 + glfwSetWindowMaximizeCallback(window, gleq_window_maximize_callback); + glfwSetWindowContentScaleCallback(window, gleq_window_content_scale_callback); +#endif +} + +GLEQDEF int gleqNextEvent(GLEQevent* event) +{ + memset(event, 0, sizeof(GLEQevent)); + + if (gleq_queue.head != gleq_queue.tail) + { + *event = gleq_queue.events[gleq_queue.tail]; + gleq_queue.tail = (gleq_queue.tail + 1) % GLEQ_CAPACITY; + } + + return event->type != GLEQ_NONE; +} + +GLEQDEF void gleqFreeEvent(GLEQevent* event) +{ +#if GLFW_VERSION_MINOR >= 1 + if (event->type == GLEQ_FILE_DROPPED) + { + while (event->file.count--) + free(event->file.paths[event->file.count]); + + free(event->file.paths); + } +#endif + + memset(event, 0, sizeof(GLEQevent)); +} + +#endif /* GLEQ_IMPLEMENTATION */ + +#endif /* GLEQ_HEADER_FILE */ diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp index 8ca9c88c5..10cd97221 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp @@ -21,12 +21,26 @@ std::string GetDataURI(const std::string& data, const std::string& mime_type) { } // namespace +AgentHandler* g_instance = nullptr; + AgentHandler::AgentHandler(CefRefPtr dev_renderer) - : dev_renderer_(dev_renderer) {} + : dev_renderer_(std::move(dev_renderer)) { + g_instance = this; +} + +AgentHandler::~AgentHandler() { + g_instance = nullptr; +} + +AgentHandler* AgentHandler::GetInstance() { + return g_instance; +} void AgentHandler::OnTitleChange(CefRefPtr browser, const CefString& title) { CEF_REQUIRE_UI_THREAD(); + if (dev_renderer_) + dev_renderer_->OnTitleChange(browser, title); } void AgentHandler::OnPaint(CefRefPtr browser, @@ -35,16 +49,16 @@ void AgentHandler::OnPaint(CefRefPtr browser, const void* buffer, int width, int height) { - - std::cout << "OnPaint" << std::endl; - if (dev_renderer_) dev_renderer_->OnPaint(browser, type, dirtyRects, buffer, width, height); } void AgentHandler::GetViewRect(CefRefPtr browser, CefRect& rect) { - CEF_REQUIRE_UI_THREAD(); - rect.Set(0, 0, 800, 600); + CEF_REQUIRE_UI_THREAD(); + + int identifier = browser->GetIdentifier(); + CefRefPtr& handle = browser_handles_[identifier]; + rect.Set(0, 0, handle->GetWidth(), handle->GetHeight()); }; void AgentHandler::OnAudioStreamPacket(CefRefPtr browser, @@ -89,11 +103,21 @@ bool AgentHandler::DoClose(CefRefPtr browser) { void AgentHandler::OnBeforeClose(CefRefPtr browser) { CEF_REQUIRE_UI_THREAD(); - if (dev_renderer_) dev_renderer_->OnBeforeClose(browser); } +void AgentHandler::OnLoadingStateChange(CefRefPtr browser, + bool isLoading, + bool canGoBack, + bool canGoForward) { + CEF_REQUIRE_UI_THREAD(); + + if (dev_renderer_) + dev_renderer_->OnLoadingStateChange(browser, isLoading, canGoBack, + canGoForward); +} + void AgentHandler::OnLoadError(CefRefPtr browser, CefRefPtr frame, ErrorCode errorCode, diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/handler.hpp b/livekit-plugins/livekit-plugins-browser/cef/src/handler.hpp index 2406c6d91..82553c816 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/handler.hpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/handler.hpp @@ -1,23 +1,39 @@ #ifndef LKCEF_HANDLER_HPP #define LKCEF_HANDLER_HPP -#include "include/cef_client.h" +#include #include "dev_renderer.hpp" -#include +#include "include/cef_client.h" +#include "include/wrapper/cef_helpers.h" -class BrowserHandle: public CefBaseRefCounted{ +class BrowserHandle : public CefBaseRefCounted { public: - BrowserHandle(std::function created_callback) : created_callback_(created_callback) {} - + BrowserHandle(std::function created_callback, int width, int height) + : created_callback_(std::move(created_callback)), + width_(width), + height_(height) {} CefRefPtr browser_ = nullptr; std::function created_callback_ = nullptr; + void SetSize(int width, int height) { + width_ = width; + height_ = height; - IMPLEMENT_REFCOUNTING(BrowserHandle); -}; + if (browser_) + browser_->GetHost()->WasResized(); + } + + int GetWidth() const { return width_; } + int GetHeight() const { return height_; } + + private: + int width_ = 0; + int height_ = 0; + IMPLEMENT_REFCOUNTING(BrowserHandle); +}; class AgentHandler : public CefClient, public CefDisplayHandler, @@ -25,9 +41,11 @@ class AgentHandler : public CefClient, public CefAudioHandler, public CefLifeSpanHandler, public CefLoadHandler { - -public: + public: AgentHandler(CefRefPtr dev_renderer); + ~AgentHandler(); + + static AgentHandler *GetInstance(); CefRefPtr GetDisplayHandler() override { return this; } CefRefPtr GetRenderHandler() override { return this; } @@ -37,27 +55,32 @@ class AgentHandler : public CefClient, // CefDisplayHandler methods void OnTitleChange(CefRefPtr browser, - const CefString &title) override; + const CefString& title) override; // CefRenderHandler methods - void OnPaint(CefRefPtr browser, PaintElementType type, - const RectList &dirtyRects, const void *buffer, int width, + void OnPaint(CefRefPtr browser, + PaintElementType type, + const RectList& dirtyRects, + const void* buffer, + int width, int height) override; - void GetViewRect(CefRefPtr browser, CefRect &rect) override; + void GetViewRect(CefRefPtr browser, CefRect& rect) override; // CefAudioHandler methods - void OnAudioStreamPacket(CefRefPtr browser, const float **data, - int frames, int64_t pts) override; + void OnAudioStreamPacket(CefRefPtr browser, + const float** data, + int frames, + int64_t pts) override; void OnAudioStreamStarted(CefRefPtr browser, - const CefAudioParameters ¶ms, + const CefAudioParameters& params, int channels) override; void OnAudioStreamStopped(CefRefPtr browser) override; void OnAudioStreamError(CefRefPtr browser, - const CefString &message) override; + const CefString& message) override; // CefLifeSpanHandler methods void OnAfterCreated(CefRefPtr browser) override; @@ -65,24 +88,38 @@ class AgentHandler : public CefClient, void OnBeforeClose(CefRefPtr browser) override; // CefLoadHandler methods - void OnLoadError(CefRefPtr browser, CefRefPtr frame, - ErrorCode errorCode, const CefString &errorText, - const CefString &failedUrl) override; - //void CloseAllBrowsers(bool force_close); + void OnLoadingStateChange(CefRefPtr browser, + bool isLoading, + bool canGoBack, + bool canGoForward) override; - static bool IsChromeRuntimeEnabled(); + void OnLoadError(CefRefPtr browser, + CefRefPtr frame, + ErrorCode errorCode, + const CefString& errorText, + const CefString& failedUrl) override; + // void CloseAllBrowsers(bool force_close); + + static bool IsChromeRuntimeEnabled(); void AddPendingHandle(CefRefPtr handle) { + CEF_REQUIRE_UI_THREAD(); pending_handles_.push_back(handle); } void RemovePendingHandle(CefRefPtr handle) { + CEF_REQUIRE_UI_THREAD(); pending_handles_.remove(handle); } -private: + CefRefPtr GetBrowserHandle(int identifier) { + CEF_REQUIRE_UI_THREAD(); + return browser_handles_[identifier]; + } + + private: std::unordered_map> browser_handles_; std::list> pending_handles_; @@ -91,4 +128,4 @@ class AgentHandler : public CefClient, IMPLEMENT_REFCOUNTING(AgentHandler); }; -#endif // LKCEF_HANDLER_HPP +#endif // LKCEF_HANDLER_HPP From 90a4e7aa885dede14b95e26e8ba3054468f5fffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?The=CC=81o=20Monnom?= Date: Sun, 18 Aug 2024 13:10:50 -0700 Subject: [PATCH 2/6] _swizzled_sendEvent --- .../cef/CMakeLists.txt | 3 +- .../cef/src/app_mac.mm | 92 +++++++------------ .../cef/src/dev_renderer.cpp | 91 +++++++++++++++++- .../cef/src/handler.cpp | 16 ++++ .../cef/src/handler.hpp | 14 +++ 5 files changed, 151 insertions(+), 65 deletions(-) diff --git a/livekit-plugins/livekit-plugins-browser/cef/CMakeLists.txt b/livekit-plugins/livekit-plugins-browser/cef/CMakeLists.txt index 0d113bd32..64d566dd7 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/CMakeLists.txt +++ b/livekit-plugins/livekit-plugins-browser/cef/CMakeLists.txt @@ -11,7 +11,8 @@ set(USE_SANDBOX OFF) # TODO(theomonnom): I don't think we want to enable sandbox # Specify the CEF distribution version. if(NOT DEFINED CEF_VERSION) - set(CEF_VERSION "122.1.10+gc902316+chromium-122.0.6261.112") + # set(CEF_VERSION "122.1.10+gc902316+chromium-122.0.6261.112") + set(CEF_VERSION "127.3.1+g6cbb30e+chromium-127.0.6533.100") endif() if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm b/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm index 3136303eb..9fa47f37a 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm +++ b/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm @@ -3,74 +3,60 @@ #include +#import +#include + #include "app.hpp" #include "handler.hpp" #include "include/cef_application_mac.h" #include "include/cef_command_line.h" #include "include/wrapper/cef_library_loader.h" -// Receives notifications from the application. -@interface AgentsAppDelegate : NSObject +BOOL g_handling_send_event = false; -- (void)createApplication:(id)object; -- (void)tryToTerminateApplication:(NSApplication*)app; -@end +@interface AgentsApplication : NSApplication + +- (BOOL)isHandlingSendEvent; +- (void)setHandlingSendEvent:(BOOL)handlingSendEvent; +- (void)_swizzled_sendEvent:(NSEvent*)event; +- (void)_swizzled_terminate:(id)sender; -// Provide the CefAppProtocol implementation required by CEF. -@interface AgentsApplication : NSApplication { - @private - BOOL handlingSendEvent_; -} @end @implementation AgentsApplication -- (BOOL)isHandlingSendEvent { - return handlingSendEvent_; -} -- (void)setHandlingSendEvent:(BOOL)handlingSendEvent { - handlingSendEvent_ = handlingSendEvent; -} - -- (void)sendEvent:(NSEvent*)event { - CefScopedSendingEvent sendingEventScoper; - [super sendEvent:event]; +// This selector is called very early during the application initialization. ++ (void)load { + // Swap NSApplication::sendEvent with _swizzled_sendEvent. + Method original = class_getInstanceMethod(self, @selector(sendEvent)); + Method swizzled = + class_getInstanceMethod(self, @selector(_swizzled_sendEvent)); + method_exchangeImplementations(original, swizzled); + + Method originalTerm = class_getInstanceMethod(self, @selector(terminate:)); + Method swizzledTerm = + class_getInstanceMethod(self, @selector(_swizzled_terminate:)); + method_exchangeImplementations(originalTerm, swizzledTerm); } -- (void)terminate:(id)sender { - AgentsAppDelegate* delegate = - static_cast([NSApp delegate]); - [delegate tryToTerminateApplication:self]; - // Return, don't exit. The application is responsible for exiting on its own. +- (BOOL)isHandlingSendEvent { + return g_handling_send_event; } -@end - -@implementation AgentsAppDelegate -// Create the application on the UI thread. -- (void)createApplication:(id)object { - [[NSBundle mainBundle] loadNibNamed:@"MainMenu" - owner:NSApp - topLevelObjects:nil]; - - // Set the delegate for application events. - [[NSApplication sharedApplication] setDelegate:self]; +- (void)setHandlingSendEvent:(BOOL)handlingSendEvent { + g_handling_send_event = handlingSendEvent; } -- (void)tryToTerminateApplication:(NSApplication*)app { +- (void)_swizzled_sendEvent:(NSEvent*)event { + CefScopedSendingEvent sendingEventScoper; + // Calls NSApplication::sendEvent due to the swizzling. + [self _swizzled_sendEvent:event]; } -- (NSApplicationTerminateReply)applicationShouldTerminate: - (NSApplication*)sender { - return NSTerminateNow; +- (void)_swizzled_terminate:(id)sender { + [self _swizzled_terminate:sender]; } -// Called when the user clicks the app dock icon while the application is -// already running. -- (BOOL)applicationShouldHandleReopen:(NSApplication*)theApplication - hasVisibleWindows:(BOOL)flag { - return NO; -} @end // Entry point function for the browser process. @@ -122,24 +108,12 @@ int RunAgentApp(CefRefPtr app) { return 1; } - // Create the application delegate. - AgentsAppDelegate* delegate = [[AgentsAppDelegate alloc] init]; - // Set as the delegate for application events. - NSApp.delegate = delegate; - - [delegate performSelectorOnMainThread:@selector(createApplication:) - withObject:nil - waitUntilDone:NO]; app->Run(); CefShutdown(); - cef_unload_library(); -#if !__has_feature(objc_arc) - [delegate release]; -#endif // !__has_feature(objc_arc) - delegate = nil; + cef_unload_library(); } // @autoreleasepool return 0; diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp index 52dd9cf32..f3c61833a 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp @@ -27,6 +27,46 @@ #define VERIFY_NO_ERROR #endif +uint32_t glfw_mods_to_cef_mods(int glfw_mods) { + uint32_t cef_flags = 0; + + if (glfw_mods & 0x0001) { // GLFW_MOD_SHIFT + cef_flags |= (1 << 1); // EVENTFLAG_SHIFT_DOWN + } + if (glfw_mods & 0x0002) { // GLFW_MOD_CONTROL + cef_flags |= (1 << 2); // EVENTFLAG_CONTROL_DOWN + } + if (glfw_mods & 0x0004) { // GLFW_MOD_ALT + cef_flags |= (1 << 3); // EVENTFLAG_ALT_DOWN + } + if (glfw_mods & 0x0008) { // GLFW_MOD_SUPER + cef_flags |= + (1 << 7); // EVENTFLAG_COMMAND_DOWN (Super key -> Command on Mac) + } + if (glfw_mods & 0x0010) { // GLFW_MOD_CAPS_LOCK + cef_flags |= (1 << 0); // EVENTFLAG_CAPS_LOCK_ON + } + if (glfw_mods & 0x0020) { // GLFW_MOD_NUM_LOCK + cef_flags |= (1 << 8); // EVENTFLAG_NUM_LOCK_ON + } + + return cef_flags; +} + +std::optional glfw_button_to_cef_button( + int button) { + switch (button) { + case GLFW_MOUSE_BUTTON_LEFT: + return CefBrowserHost::MouseButtonType::MBT_LEFT; + case GLFW_MOUSE_BUTTON_MIDDLE: + return CefBrowserHost::MouseButtonType::MBT_MIDDLE; + case GLFW_MOUSE_BUTTON_RIGHT: + return CefBrowserHost::MouseButtonType::MBT_RIGHT; + default: + return std::nullopt; + } +} + static void glfw_error_callback(int error, const char* description) { fprintf(stderr, "GLFW Error %d: %s\n", error, description); } @@ -318,15 +358,56 @@ void DevRenderer::Run() { while (gleqNextEvent(&event)) { switch (event.type) { case GLEQ_CURSOR_MOVED: - + case GLEQ_BUTTON_PRESSED: + case GLEQ_BUTTON_RELEASED: if (focused_browser) { CefMouseEvent cef_event; - cef_event.x = event.pos.x - browser_view_x; - cef_event.y = event.pos.y - browser_view_y; - focused_browser->browser->GetHost()->SendMouseMoveEvent(cef_event, - false); + + if (event.type == GLEQ_CURSOR_MOVED) { + cef_event.x = event.pos.x - browser_view_x; + cef_event.y = event.pos.y - browser_view_y; + focused_browser->browser->GetHost()->SendMouseMoveEvent(cef_event, + false); + } else { + double xpos, ypos; + glfwGetCursorPos(window_, &xpos, &ypos); + cef_event.x = static_cast(xpos) - browser_view_x; + cef_event.y = static_cast(ypos) - browser_view_y; + cef_event.modifiers = glfw_mods_to_cef_mods(event.mouse.mods); + + std::optional cef_button = + glfw_button_to_cef_button(event.mouse.button); + + if (cef_button.has_value()) { + focused_browser->browser->GetHost()->SendMouseClickEvent( + cef_event, cef_button.value(), + event.type == GLEQ_BUTTON_RELEASED, 1); + } + } } + break; + + /* + * + * +if (button >= 0 && button <= 2) { // GLFW only supports buttons 0-2 (left, +right, middle) CefMouseEvent cef_event; double xpos, ypos; + glfwGetCursorPos(window, &xpos, &ypos); + cef_event.x = static_cast(xpos); + cef_event.y = static_cast(ypos); + auto browser = GetBrowserHandleFromWindow(window); + + if (action == GLFW_PRESS) { + browser->GetHost()->SendMouseClickEvent(cef_event, + static_cast(button), +false, 1); } else if (action == GLFW_RELEASE) { +browser->GetHost()->SendMouseClickEvent(cef_event, + static_cast(button), +true, 1); + } + } + */ break; default: break; diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp index 10cd97221..d8020424f 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp @@ -77,6 +77,22 @@ void AgentHandler::OnAudioStreamStopped(CefRefPtr browser) {} void AgentHandler::OnAudioStreamError(CefRefPtr browser, const CefString& message) {} + +bool AgentHandler::OnBeforePopup(CefRefPtr browser, + CefRefPtr frame, + const CefString& target_url, + const CefString& target_frame_name, + WindowOpenDisposition target_disposition, + bool user_gesture, + const CefPopupFeatures& popupFeatures, + CefWindowInfo& windowInfo, + CefRefPtr& client, + CefBrowserSettings& settings, + CefRefPtr& extra_info, + bool* no_javascript_access) { + return true; +} + void AgentHandler::OnAfterCreated(CefRefPtr browser) { CEF_REQUIRE_UI_THREAD(); diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/handler.hpp b/livekit-plugins/livekit-plugins-browser/cef/src/handler.hpp index 82553c816..92236d970 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/handler.hpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/handler.hpp @@ -83,6 +83,20 @@ class AgentHandler : public CefClient, const CefString& message) override; // CefLifeSpanHandler methods + + bool OnBeforePopup(CefRefPtr browser, + CefRefPtr frame, + const CefString& target_url, + const CefString& target_frame_name, + WindowOpenDisposition target_disposition, + bool user_gesture, + const CefPopupFeatures& popupFeatures, + CefWindowInfo& windowInfo, + CefRefPtr& client, + CefBrowserSettings& settings, + CefRefPtr& extra_info, + bool* no_javascript_access) override; + void OnAfterCreated(CefRefPtr browser) override; bool DoClose(CefRefPtr browser) override; void OnBeforeClose(CefRefPtr browser) override; From 1cb72a1835327140def3d95a0c2b44e18ae664f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?The=CC=81o=20Monnom?= Date: Sun, 18 Aug 2024 13:54:40 -0700 Subject: [PATCH 3/6] temp crash fix --- .../livekit-plugins-browser/cef/src/app.hpp | 2 ++ .../cef/src/app_mac.mm | 11 +++++---- .../cef/src/dev_renderer.cpp | 23 ------------------- .../cef/src/handler.cpp | 1 + 4 files changed, 10 insertions(+), 27 deletions(-) diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/app.hpp b/livekit-plugins/livekit-plugins-browser/cef/src/app.hpp index b3c812173..d5a8b70ed 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/app.hpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/app.hpp @@ -34,6 +34,8 @@ class AgentApp : public CefApp, public CefBrowserProcessHandler { int Run(); + bool IsDevMode() const { return dev_mode_; } + private: IMPLEMENT_REFCOUNTING(AgentApp); diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm b/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm index 9fa47f37a..7756a0209 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm +++ b/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm @@ -14,7 +14,7 @@ BOOL g_handling_send_event = false; -@interface AgentsApplication : NSApplication +@interface NSApplication (AgentsApplication) - (BOOL)isHandlingSendEvent; - (void)setHandlingSendEvent:(BOOL)handlingSendEvent; @@ -23,7 +23,7 @@ - (void)_swizzled_terminate:(id)sender; @end -@implementation AgentsApplication +@implementation NSApplication (AgentsApplication) // This selector is called very early during the application initialization. + (void)load { @@ -48,6 +48,7 @@ - (void)setHandlingSendEvent:(BOOL)handlingSendEvent { } - (void)_swizzled_sendEvent:(NSEvent*)event { + NSLog(@"_swizzled_sendEvent"); CefScopedSendingEvent sendingEventScoper; // Calls NSApplication::sendEvent due to the swizzling. [self _swizzled_sendEvent:event]; @@ -64,12 +65,12 @@ int RunAgentApp(CefRefPtr app) { CefMainArgs main_args(0, nullptr); @autoreleasepool { - [AgentsApplication sharedApplication]; + //[NSApplication sharedApplication]; // If there was an invocation to NSApp prior to this method, then the NSApp // will not be a AgentsApplication, but will instead be an NSApplication. // This is undesirable and we must enforce that this doesn't happen. - CHECK([NSApp isKindOfClass:[AgentsApplication class]]); + //CHECK([NSApp isKindOfClass:[NSApplication class]]); std::string framework_path = "/Users/theomonnom/livekit/agents/livekit-plugins/" @@ -90,6 +91,8 @@ int RunAgentApp(CefRefPtr app) { } CefSettings settings{}; + settings.chrome_runtime = true; + settings.external_message_pump = app->IsDevMode(); // settings.remote_debugging_port = 8088; CefString(&settings.framework_dir_path).FromString(framework_path); CefString(&settings.main_bundle_path).FromString(main_bundle_path); diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp index f3c61833a..b3f08bab9 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp @@ -386,29 +386,6 @@ void DevRenderer::Run() { } } break; - - /* - * - * -if (button >= 0 && button <= 2) { // GLFW only supports buttons 0-2 (left, -right, middle) CefMouseEvent cef_event; double xpos, ypos; - glfwGetCursorPos(window, &xpos, &ypos); - cef_event.x = static_cast(xpos); - cef_event.y = static_cast(ypos); - - auto browser = GetBrowserHandleFromWindow(window); - - if (action == GLFW_PRESS) { - browser->GetHost()->SendMouseClickEvent(cef_event, - static_cast(button), -false, 1); } else if (action == GLFW_RELEASE) { -browser->GetHost()->SendMouseClickEvent(cef_event, - static_cast(button), -true, 1); - } - } - */ - break; default: break; } diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp index d8020424f..9ada4e79d 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/handler.cpp @@ -90,6 +90,7 @@ bool AgentHandler::OnBeforePopup(CefRefPtr browser, CefBrowserSettings& settings, CefRefPtr& extra_info, bool* no_javascript_access) { + browser->GetMainFrame()->LoadURL(target_url); return true; } From ba07de2cd94405b48065516f8318bec965b34324 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?The=CC=81o=20Monnom?= Date: Sun, 18 Aug 2024 14:19:19 -0700 Subject: [PATCH 4/6] forward scroll events --- .../cef/src/app_mac.mm | 2 +- .../cef/src/dev_renderer.cpp | 54 ++++++------------- 2 files changed, 17 insertions(+), 39 deletions(-) diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm b/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm index 7756a0209..848e53802 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm +++ b/livekit-plugins/livekit-plugins-browser/cef/src/app_mac.mm @@ -27,6 +27,7 @@ @implementation NSApplication (AgentsApplication) // This selector is called very early during the application initialization. + (void)load { + NSLog(@"AgentsApplication::load"); // Swap NSApplication::sendEvent with _swizzled_sendEvent. Method original = class_getInstanceMethod(self, @selector(sendEvent)); Method swizzled = @@ -48,7 +49,6 @@ - (void)setHandlingSendEvent:(BOOL)handlingSendEvent { } - (void)_swizzled_sendEvent:(NSEvent*)event { - NSLog(@"_swizzled_sendEvent"); CefScopedSendingEvent sendingEventScoper; // Calls NSApplication::sendEvent due to the swizzling. [self _swizzled_sendEvent:event]; diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp index b3f08bab9..fc195fc87 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp @@ -27,7 +27,7 @@ #define VERIFY_NO_ERROR #endif -uint32_t glfw_mods_to_cef_mods(int glfw_mods) { +static uint32_t glfw_mods_to_cef_mods(int glfw_mods) { uint32_t cef_flags = 0; if (glfw_mods & 0x0001) { // GLFW_MOD_SHIFT @@ -53,7 +53,7 @@ uint32_t glfw_mods_to_cef_mods(int glfw_mods) { return cef_flags; } -std::optional glfw_button_to_cef_button( +static std::optional glfw_button_to_cef_button( int button) { switch (button) { case GLFW_MOUSE_BUTTON_LEFT: @@ -103,45 +103,10 @@ static_cast(glfwGetWindowUserPointer(window)); browser->GetHost()->SendKeyEvent(cef_event); }*/ -/*static void glfw_mouse_button_callback(GLFWwindow* window, int button, int -action, int mods) { DevRenderer* dev_renderer = -static_cast(glfwGetWindowUserPointer(window)); - - if (button >= 0 && button <= 2) { // GLFW only supports buttons 0-2 (left, -right, middle) CefMouseEvent cef_event; double xpos, ypos; - glfwGetCursorPos(window, &xpos, &ypos); - cef_event.x = static_cast(xpos); - cef_event.y = static_cast(ypos); - - auto browser = GetBrowserHandleFromWindow(window); - - if (action == GLFW_PRESS) { - browser->GetHost()->SendMouseClickEvent(cef_event, -static_cast(button), false, 1); } else if (action == -GLFW_RELEASE) { browser->GetHost()->SendMouseClickEvent(cef_event, -static_cast(button), true, 1); - } - } -} -// Mouse move callback -static void glfw_cursor_position_callback(GLFWwindow* window, double xpos, -double ypos) { DevRenderer* dev_renderer = -static_cast(glfwGetWindowUserPointer(window)); - DevRenderer::BrowserData* data = dev_renderer->getSelectedBrowserData(); - if (!data) return; - CefMouseEvent cef_event; - cef_event.x = static_cast(xpos); - cef_event.y = static_cast(ypos); - - // Translate the coordinate to the ImGUI window - - data->browser->GetHost()->SendMouseMoveEvent(cef_event, false); -} - -// Scroll callback +/* Scroll callback static void glfw_scroll_callback(GLFWwindow* window, double xoffset, double yoffset) { DevRenderer* dev_renderer = static_cast(glfwGetWindowUserPointer(window)); @@ -359,6 +324,7 @@ void DevRenderer::Run() { switch (event.type) { case GLEQ_CURSOR_MOVED: case GLEQ_BUTTON_PRESSED: + case GLEQ_SCROLLED: case GLEQ_BUTTON_RELEASED: if (focused_browser) { CefMouseEvent cef_event; @@ -368,6 +334,18 @@ void DevRenderer::Run() { cef_event.y = event.pos.y - browser_view_y; focused_browser->browser->GetHost()->SendMouseMoveEvent(cef_event, false); + } else if (event.type == GLEQ_SCROLLED) { + double xpos, ypos; + glfwGetCursorPos(window_, &xpos, &ypos); + cef_event.x = static_cast(xpos) - browser_view_x; + cef_event.y = static_cast(ypos) - browser_view_y; + + static const int scrollbarPixelsPerTick = 20; + int scroll_x = static_cast(event.scroll.x * scrollbarPixelsPerTick); + int scroll_y = static_cast(event.scroll.y * scrollbarPixelsPerTick); + + focused_browser->browser->GetHost()->SendMouseWheelEvent( + cef_event, scroll_x, scroll_y); } else { double xpos, ypos; glfwGetCursorPos(window_, &xpos, &ypos); From 51067ebbf834043a9d64b8dd8f0160eb195f3340 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?The=CC=81o=20Monnom?= Date: Sun, 18 Aug 2024 14:20:33 -0700 Subject: [PATCH 5/6] fmt --- .../livekit-plugins-browser/cef/src/dev_renderer.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp index fc195fc87..aff81d669 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp @@ -103,9 +103,6 @@ static_cast(glfwGetWindowUserPointer(window)); browser->GetHost()->SendKeyEvent(cef_event); }*/ - - - /* Scroll callback static void glfw_scroll_callback(GLFWwindow* window, double xoffset, double yoffset) { DevRenderer* dev_renderer = @@ -341,8 +338,10 @@ void DevRenderer::Run() { cef_event.y = static_cast(ypos) - browser_view_y; static const int scrollbarPixelsPerTick = 20; - int scroll_x = static_cast(event.scroll.x * scrollbarPixelsPerTick); - int scroll_y = static_cast(event.scroll.y * scrollbarPixelsPerTick); + int scroll_x = + static_cast(event.scroll.x * scrollbarPixelsPerTick); + int scroll_y = + static_cast(event.scroll.y * scrollbarPixelsPerTick); focused_browser->browser->GetHost()->SendMouseWheelEvent( cef_event, scroll_x, scroll_y); From 4ac50b8ae6949f3fd1ae384a58267c398424270b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?The=CC=81o=20Monnom?= Date: Sun, 18 Aug 2024 15:15:36 -0700 Subject: [PATCH 6/6] keyboard inputs --- .../cef/src/dev_renderer.cpp | 248 ++++++-- .../cef/src/keyboard_codes.h | 528 ++++++++++++++++++ 2 files changed, 728 insertions(+), 48 deletions(-) create mode 100644 livekit-plugins/livekit-plugins-browser/cef/src/keyboard_codes.h diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp index aff81d669..ab22e1748 100644 --- a/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp +++ b/livekit-plugins/livekit-plugins-browser/cef/src/dev_renderer.cpp @@ -11,6 +11,7 @@ #include "imgui_stdlib.h" #include "include/cef_app.h" #include "include/wrapper/cef_helpers.h" +#include "keyboard_codes.h" #define GLEQ_IMPLEMENTATION #define GLEQ_STATIC @@ -27,6 +28,172 @@ #define VERIFY_NO_ERROR #endif +int glfw_key_to_cef_key(int glfwKey) { + switch (glfwKey) { + case GLFW_KEY_SPACE: + return WebCore::VK_SPACE; + case GLFW_KEY_APOSTROPHE: + return WebCore::VK_OEM_7; + case GLFW_KEY_COMMA: + return WebCore::VK_OEM_COMMA; + case GLFW_KEY_MINUS: + return WebCore::VK_OEM_MINUS; + case GLFW_KEY_PERIOD: + return WebCore::VK_OEM_PERIOD; + case GLFW_KEY_SLASH: + return WebCore::VK_OEM_2; + case GLFW_KEY_0: + return WebCore::VK_0; + case GLFW_KEY_1: + return WebCore::VK_1; + case GLFW_KEY_2: + return WebCore::VK_2; + case GLFW_KEY_3: + return WebCore::VK_3; + case GLFW_KEY_4: + return WebCore::VK_4; + case GLFW_KEY_5: + return WebCore::VK_5; + case GLFW_KEY_6: + return WebCore::VK_6; + case GLFW_KEY_7: + return WebCore::VK_7; + case GLFW_KEY_8: + return WebCore::VK_8; + case GLFW_KEY_9: + return WebCore::VK_9; + case GLFW_KEY_SEMICOLON: + return WebCore::VK_OEM_1; + case GLFW_KEY_EQUAL: + return WebCore::VK_OEM_PLUS; + case GLFW_KEY_A: + return WebCore::VK_A; + case GLFW_KEY_B: + return WebCore::VK_B; + case GLFW_KEY_C: + return WebCore::VK_C; + case GLFW_KEY_D: + return WebCore::VK_D; + case GLFW_KEY_E: + return WebCore::VK_E; + case GLFW_KEY_F: + return WebCore::VK_F; + case GLFW_KEY_G: + return WebCore::VK_G; + case GLFW_KEY_H: + return WebCore::VK_H; + case GLFW_KEY_I: + return WebCore::VK_I; + case GLFW_KEY_J: + return WebCore::VK_J; + case GLFW_KEY_K: + return WebCore::VK_K; + case GLFW_KEY_L: + return WebCore::VK_L; + case GLFW_KEY_M: + return WebCore::VK_M; + case GLFW_KEY_N: + return WebCore::VK_N; + case GLFW_KEY_O: + return WebCore::VK_O; + case GLFW_KEY_P: + return WebCore::VK_P; + case GLFW_KEY_Q: + return WebCore::VK_Q; + case GLFW_KEY_R: + return WebCore::VK_R; + case GLFW_KEY_S: + return WebCore::VK_S; + case GLFW_KEY_T: + return WebCore::VK_T; + case GLFW_KEY_U: + return WebCore::VK_U; + case GLFW_KEY_V: + return WebCore::VK_V; + case GLFW_KEY_W: + return WebCore::VK_W; + case GLFW_KEY_X: + return WebCore::VK_X; + case GLFW_KEY_Y: + return WebCore::VK_Y; + case GLFW_KEY_Z: + return WebCore::VK_Z; + case GLFW_KEY_LEFT_BRACKET: + return WebCore::VK_OEM_4; + case GLFW_KEY_BACKSLASH: + return WebCore::VK_OEM_5; + case GLFW_KEY_RIGHT_BRACKET: + return WebCore::VK_OEM_6; + case GLFW_KEY_GRAVE_ACCENT: + return WebCore::VK_OEM_3; + case GLFW_KEY_ESCAPE: + return WebCore::VK_ESCAPE; + case GLFW_KEY_ENTER: + return WebCore::VK_RETURN; + case GLFW_KEY_TAB: + return WebCore::VK_TAB; + case GLFW_KEY_BACKSPACE: + return WebCore::VK_BACK; + case GLFW_KEY_INSERT: + return WebCore::VK_INSERT; + case GLFW_KEY_DELETE: + return WebCore::VK_DELETE; + case GLFW_KEY_RIGHT: + return WebCore::VK_RIGHT; + case GLFW_KEY_LEFT: + return WebCore::VK_LEFT; + case GLFW_KEY_DOWN: + return WebCore::VK_DOWN; + case GLFW_KEY_UP: + return WebCore::VK_UP; + case GLFW_KEY_PAGE_UP: + return WebCore::VK_PRIOR; + case GLFW_KEY_PAGE_DOWN: + return WebCore::VK_NEXT; + case GLFW_KEY_HOME: + return WebCore::VK_HOME; + case GLFW_KEY_END: + return WebCore::VK_END; + case GLFW_KEY_CAPS_LOCK: + return WebCore::VK_CAPITAL; + case GLFW_KEY_SCROLL_LOCK: + return WebCore::VK_SCROLL; + case GLFW_KEY_NUM_LOCK: + return WebCore::VK_NUMLOCK; + case GLFW_KEY_PRINT_SCREEN: + return WebCore::VK_SNAPSHOT; + case GLFW_KEY_PAUSE: + return WebCore::VK_PAUSE; + case GLFW_KEY_F1: + return WebCore::VK_F1; + case GLFW_KEY_F2: + return WebCore::VK_F2; + case GLFW_KEY_F3: + return WebCore::VK_F3; + case GLFW_KEY_F4: + return WebCore::VK_F4; + case GLFW_KEY_F5: + return WebCore::VK_F5; + case GLFW_KEY_F6: + return WebCore::VK_F6; + case GLFW_KEY_F7: + return WebCore::VK_F7; + case GLFW_KEY_F8: + return WebCore::VK_F8; + case GLFW_KEY_F9: + return WebCore::VK_F9; + case GLFW_KEY_F10: + return WebCore::VK_F10; + case GLFW_KEY_F11: + return WebCore::VK_F11; + case GLFW_KEY_F12: + return WebCore::VK_F12; + // Add more cases as needed + default: + return WebCore::VK_UNKNOWN; + } +} + static uint32_t glfw_mods_to_cef_mods(int glfw_mods) { uint32_t cef_flags = 0; @@ -71,54 +238,6 @@ static void glfw_error_callback(int error, const char* description) { fprintf(stderr, "GLFW Error %d: %s\n", error, description); } -/*static void glfw_key_callback(GLFWwindow* window, int key, int scancode, int -action, int mods) { DevRenderer* dev_renderer = -static_cast(glfwGetWindowUserPointer(window)); - - if (action == GLFW_PRESS || action == GLFW_RELEASE) { - // Handle key event forwarding to CEF - CefKeyEvent cef_event; - cef_event.windows_key_code = key; - cef_event.native_key_code = scancode; - cef_event.is_system_key = false; - cef_event.type = (action == GLFW_PRESS) ? KEYEVENT_RAWKEYDOWN : -KEYEVENT_KEYUP; cef_event.modifiers = mods; - - auto browser = GetBrowserHandleFromWindow(window); - browser->GetHost()->SendKeyEvent(cef_event); - } -} - - - -void glfw_char_callback(GLFWwindow* window, unsigned int codepoint) { - DevRenderer* dev_renderer = -static_cast(glfwGetWindowUserPointer(window)); - - CefKeyEvent cef_event; - cef_event.character = codepoint; - cef_event.type = KEYEVENT_CHAR; - - auto browser = GetBrowserHandleFromWindow(window); - browser->GetHost()->SendKeyEvent(cef_event); -}*/ - -/* Scroll callback -static void glfw_scroll_callback(GLFWwindow* window, double xoffset, double -yoffset) { DevRenderer* dev_renderer = -static_cast(glfwGetWindowUserPointer(window)); - - CefMouseEvent cef_event; - double xpos, ypos; - glfwGetCursorPos(window, &xpos, &ypos); - cef_event.x = static_cast(xpos); - cef_event.y = static_cast(ypos); - - auto browser = GetBrowserHandleFromWindow(window); - browser->GetHost()->SendMouseWheelEvent(cef_event, static_cast(xoffset * -100), static_cast(yoffset * 100)); -}*/ - DevRenderer::DevRenderer() {} void DevRenderer::OnTitleChange(CefRefPtr browser, @@ -306,6 +425,7 @@ void DevRenderer::Run() { focused_browser = &data; browser_view_x = static_cast(cursor_pos.x); browser_view_y = static_cast(cursor_pos.y); + data.browser->GetHost()->SetFocus(true); } // Render the browser tex @@ -363,6 +483,38 @@ void DevRenderer::Run() { } } break; + case GLEQ_KEY_PRESSED: + case GLEQ_KEY_RELEASED: + if (focused_browser) { + CefKeyEvent cef_event; + cef_event.windows_key_code = + glfw_key_to_cef_key(event.keyboard.key); + cef_event.native_key_code = event.keyboard.scancode; + cef_event.modifiers = glfw_mods_to_cef_mods(event.keyboard.mods); + cef_event.is_system_key = false; + + if (event.type == GLEQ_KEY_PRESSED) { + cef_event.type = KEYEVENT_RAWKEYDOWN; + focused_browser->browser->GetHost()->SendKeyEvent(cef_event); + } else { + cef_event.type = KEYEVENT_KEYUP; + focused_browser->browser->GetHost()->SendKeyEvent(cef_event); + } + } + break; + case GLEQ_CODEPOINT_INPUT: + if (focused_browser) { + CefKeyEvent cef_event; + cef_event.type = KEYEVENT_CHAR; + cef_event.windows_key_code = 0; + cef_event.native_key_code = 0; + cef_event.modifiers = 0; + cef_event.is_system_key = false; + cef_event.unmodified_character = event.codepoint; + cef_event.character = event.codepoint; + focused_browser->browser->GetHost()->SendKeyEvent(cef_event); + } + break; default: break; } diff --git a/livekit-plugins/livekit-plugins-browser/cef/src/keyboard_codes.h b/livekit-plugins/livekit-plugins-browser/cef/src/keyboard_codes.h new file mode 100644 index 000000000..5a3b67e82 --- /dev/null +++ b/livekit-plugins/livekit-plugins-browser/cef/src/keyboard_codes.h @@ -0,0 +1,528 @@ +#ifndef LKCEF_KEYBOARD_CODES_H +#define LKCEF_KEYBOARD_CODES_H + +namespace WebCore { +// VK_LBUTTON (01) Left mouse button +// VK_RBUTTON (02) Right mouse button +// VK_CANCEL (03) Control-break processing +// VK_MBUTTON (04) Middle mouse button (three-button mouse) +// VK_XBUTTON1 (05) +// VK_XBUTTON2 (06) + +// VK_BACK (08) BACKSPACE key +const int VK_BACK = 0x08; + +// VK_TAB (09) TAB key +const int VK_TAB = 0x09; + +// VK_CLEAR (0C) CLEAR key +const int VK_CLEAR = 0x0C; + +// VK_RETURN (0D) +const int VK_RETURN = 0x0D; + +// VK_SHIFT (10) SHIFT key +const int VK_SHIFT = 0x10; + +// VK_CONTROL (11) CTRL key +const int VK_CONTROL = 0x11; + +// VK_MENU (12) ALT key +const int VK_MENU = 0x12; + +// VK_PAUSE (13) PAUSE key +const int VK_PAUSE = 0x13; + +// VK_CAPITAL (14) CAPS LOCK key +const int VK_CAPITAL = 0x14; + +// VK_KANA (15) Input Method Editor (IME) Kana mode +const int VK_KANA = 0x15; + +// VK_HANGUEL (15) IME Hanguel mode (maintained for compatibility; use +// VK_HANGUL) VK_HANGUL (15) IME Hangul mode +const int VK_HANGUL = 0x15; + +// VK_JUNJA (17) IME Junja mode +const int VK_JUNJA = 0x17; + +// VK_FINAL (18) IME final mode +const int VK_FINAL = 0x18; + +// VK_HANJA (19) IME Hanja mode +const int VK_HANJA = 0x19; + +// VK_KANJI (19) IME Kanji mode +const int VK_KANJI = 0x19; + +// VK_ESCAPE (1B) ESC key +const int VK_ESCAPE = 0x1B; + +// VK_CONVERT (1C) IME convert +const int VK_CONVERT = 0x1C; + +// VK_NONCONVERT (1D) IME nonconvert +const int VK_NONCONVERT = 0x1D; + +// VK_ACCEPT (1E) IME accept +const int VK_ACCEPT = 0x1E; + +// VK_MODECHANGE (1F) IME mode change request +const int VK_MODECHANGE = 0x1F; + +// VK_SPACE (20) SPACEBAR +const int VK_SPACE = 0x20; + +// VK_PRIOR (21) PAGE UP key +const int VK_PRIOR = 0x21; + +// VK_NEXT (22) PAGE DOWN key +const int VK_NEXT = 0x22; + +// VK_END (23) END key +const int VK_END = 0x23; + +// VK_HOME (24) HOME key +const int VK_HOME = 0x24; + +// VK_LEFT (25) LEFT ARROW key +const int VK_LEFT = 0x25; + +// VK_UP (26) UP ARROW key +const int VK_UP = 0x26; + +// VK_RIGHT (27) RIGHT ARROW key +const int VK_RIGHT = 0x27; + +// VK_DOWN (28) DOWN ARROW key +const int VK_DOWN = 0x28; + +// VK_SELECT (29) SELECT key +const int VK_SELECT = 0x29; + +// VK_PRINT (2A) PRINT key +const int VK_PRINT = 0x2A; + +// VK_EXECUTE (2B) EXECUTE key +const int VK_EXECUTE = 0x2B; + +// VK_SNAPSHOT (2C) PRINT SCREEN key +const int VK_SNAPSHOT = 0x2C; + +// VK_INSERT (2D) INS key +const int VK_INSERT = 0x2D; + +// VK_DELETE (2E) DEL key +const int VK_DELETE = 0x2E; + +// VK_HELP (2F) HELP key +const int VK_HELP = 0x2F; + +// (30) 0 key +const int VK_0 = 0x30; + +// (31) 1 key +const int VK_1 = 0x31; + +// (32) 2 key +const int VK_2 = 0x32; + +// (33) 3 key +const int VK_3 = 0x33; + +// (34) 4 key +const int VK_4 = 0x34; + +// (35) 5 key; + +const int VK_5 = 0x35; + +// (36) 6 key +const int VK_6 = 0x36; + +// (37) 7 key +const int VK_7 = 0x37; + +// (38) 8 key +const int VK_8 = 0x38; + +// (39) 9 key +const int VK_9 = 0x39; + +// (41) A key +const int VK_A = 0x41; + +// (42) B key +const int VK_B = 0x42; + +// (43) C key +const int VK_C = 0x43; + +// (44) D key +const int VK_D = 0x44; + +// (45) E key +const int VK_E = 0x45; + +// (46) F key +const int VK_F = 0x46; + +// (47) G key +const int VK_G = 0x47; + +// (48) H key +const int VK_H = 0x48; + +// (49) I key +const int VK_I = 0x49; + +// (4A) J key +const int VK_J = 0x4A; + +// (4B) K key +const int VK_K = 0x4B; + +// (4C) L key +const int VK_L = 0x4C; + +// (4D) M key +const int VK_M = 0x4D; + +// (4E) N key +const int VK_N = 0x4E; + +// (4F) O key +const int VK_O = 0x4F; + +// (50) P key +const int VK_P = 0x50; + +// (51) Q key +const int VK_Q = 0x51; + +// (52) R key +const int VK_R = 0x52; + +// (53) S key +const int VK_S = 0x53; + +// (54) T key +const int VK_T = 0x54; + +// (55) U key +const int VK_U = 0x55; + +// (56) V key +const int VK_V = 0x56; + +// (57) W key +const int VK_W = 0x57; + +// (58) X key +const int VK_X = 0x58; + +// (59) Y key +const int VK_Y = 0x59; + +// (5A) Z key +const int VK_Z = 0x5A; + +// VK_LWIN (5B) Left Windows key (Microsoft Natural keyboard) +const int VK_LWIN = 0x5B; + +// VK_RWIN (5C) Right Windows key (Natural keyboard) +const int VK_RWIN = 0x5C; + +// VK_APPS (5D) Applications key (Natural keyboard) +const int VK_APPS = 0x5D; + +// VK_SLEEP (5F) Computer Sleep key +const int VK_SLEEP = 0x5F; + +// VK_NUMPAD0 (60) Numeric keypad 0 key +const int VK_NUMPAD0 = 0x60; + +// VK_NUMPAD1 (61) Numeric keypad 1 key +const int VK_NUMPAD1 = 0x61; + +// VK_NUMPAD2 (62) Numeric keypad 2 key +const int VK_NUMPAD2 = 0x62; + +// VK_NUMPAD3 (63) Numeric keypad 3 key +const int VK_NUMPAD3 = 0x63; + +// VK_NUMPAD4 (64) Numeric keypad 4 key +const int VK_NUMPAD4 = 0x64; + +// VK_NUMPAD5 (65) Numeric keypad 5 key +const int VK_NUMPAD5 = 0x65; + +// VK_NUMPAD6 (66) Numeric keypad 6 key +const int VK_NUMPAD6 = 0x66; + +// VK_NUMPAD7 (67) Numeric keypad 7 key +const int VK_NUMPAD7 = 0x67; + +// VK_NUMPAD8 (68) Numeric keypad 8 key +const int VK_NUMPAD8 = 0x68; + +// VK_NUMPAD9 (69) Numeric keypad 9 key +const int VK_NUMPAD9 = 0x69; + +// VK_MULTIPLY (6A) Multiply key +const int VK_MULTIPLY = 0x6A; + +// VK_ADD (6B) Add key +const int VK_ADD = 0x6B; + +// VK_SEPARATOR (6C) Separator key +const int VK_SEPARATOR = 0x6C; + +// VK_SUBTRACT (6D) Subtract key +const int VK_SUBTRACT = 0x6D; + +// VK_DECIMAL (6E) Decimal key +const int VK_DECIMAL = 0x6E; + +// VK_DIVIDE (6F) Divide key +const int VK_DIVIDE = 0x6F; + +// VK_F1 (70) F1 key +const int VK_F1 = 0x70; + +// VK_F2 (71) F2 key +const int VK_F2 = 0x71; + +// VK_F3 (72) F3 key +const int VK_F3 = 0x72; + +// VK_F4 (73) F4 key +const int VK_F4 = 0x73; + +// VK_F5 (74) F5 key +const int VK_F5 = 0x74; + +// VK_F6 (75) F6 key +const int VK_F6 = 0x75; + +// VK_F7 (76) F7 key +const int VK_F7 = 0x76; + +// VK_F8 (77) F8 key +const int VK_F8 = 0x77; + +// VK_F9 (78) F9 key +const int VK_F9 = 0x78; + +// VK_F10 (79) F10 key +const int VK_F10 = 0x79; + +// VK_F11 (7A) F11 key +const int VK_F11 = 0x7A; + +// VK_F12 (7B) F12 key +const int VK_F12 = 0x7B; + +// VK_F13 (7C) F13 key +const int VK_F13 = 0x7C; + +// VK_F14 (7D) F14 key +const int VK_F14 = 0x7D; + +// VK_F15 (7E) F15 key +const int VK_F15 = 0x7E; + +// VK_F16 (7F) F16 key +const int VK_F16 = 0x7F; + +// VK_F17 (80H) F17 key +const int VK_F17 = 0x80; + +// VK_F18 (81H) F18 key +const int VK_F18 = 0x81; + +// VK_F19 (82H) F19 key +const int VK_F19 = 0x82; + +// VK_F20 (83H) F20 key +const int VK_F20 = 0x83; + +// VK_F21 (84H) F21 key +const int VK_F21 = 0x84; + +// VK_F22 (85H) F22 key +const int VK_F22 = 0x85; + +// VK_F23 (86H) F23 key +const int VK_F23 = 0x86; + +// VK_F24 (87H) F24 key +const int VK_F24 = 0x87; + +// VK_NUMLOCK (90) NUM LOCK key +const int VK_NUMLOCK = 0x90; + +// VK_SCROLL (91) SCROLL LOCK key +const int VK_SCROLL = 0x91; + +// VK_LSHIFT (A0) Left SHIFT key +const int VK_LSHIFT = 0xA0; + +// VK_RSHIFT (A1) Right SHIFT key +const int VK_RSHIFT = 0xA1; + +// VK_LCONTROL (A2) Left CONTROL key +const int VK_LCONTROL = 0xA2; + +// VK_RCONTROL (A3) Right CONTROL key +const int VK_RCONTROL = 0xA3; + +// VK_LMENU (A4) Left MENU key +const int VK_LMENU = 0xA4; + +// VK_RMENU (A5) Right MENU key +const int VK_RMENU = 0xA5; + +// VK_BROWSER_BACK (A6) Windows 2000/XP: Browser Back key +const int VK_BROWSER_BACK = 0xA6; + +// VK_BROWSER_FORWARD (A7) Windows 2000/XP: Browser Forward key +const int VK_BROWSER_FORWARD = 0xA7; + +// VK_BROWSER_REFRESH (A8) Windows 2000/XP: Browser Refresh key +const int VK_BROWSER_REFRESH = 0xA8; + +// VK_BROWSER_STOP (A9) Windows 2000/XP: Browser Stop key +const int VK_BROWSER_STOP = 0xA9; + +// VK_BROWSER_SEARCH (AA) Windows 2000/XP: Browser Search key +const int VK_BROWSER_SEARCH = 0xAA; + +// VK_BROWSER_FAVORITES (AB) Windows 2000/XP: Browser Favorites key +const int VK_BROWSER_FAVORITES = 0xAB; + +// VK_BROWSER_HOME (AC) Windows 2000/XP: Browser Start and Home key +const int VK_BROWSER_HOME = 0xAC; + +// VK_VOLUME_MUTE (AD) Windows 2000/XP: Volume Mute key +const int VK_VOLUME_MUTE = 0xAD; + +// VK_VOLUME_DOWN (AE) Windows 2000/XP: Volume Down key +const int VK_VOLUME_DOWN = 0xAE; + +// VK_VOLUME_UP (AF) Windows 2000/XP: Volume Up key +const int VK_VOLUME_UP = 0xAF; + +// VK_MEDIA_NEXT_TRACK (B0) Windows 2000/XP: Next Track key +const int VK_MEDIA_NEXT_TRACK = 0xB0; + +// VK_MEDIA_PREV_TRACK (B1) Windows 2000/XP: Previous Track key +const int VK_MEDIA_PREV_TRACK = 0xB1; + +// VK_MEDIA_STOP (B2) Windows 2000/XP: Stop Media key +const int VK_MEDIA_STOP = 0xB2; + +// VK_MEDIA_PLAY_PAUSE (B3) Windows 2000/XP: Play/Pause Media key +const int VK_MEDIA_PLAY_PAUSE = 0xB3; + +// VK_LAUNCH_MAIL (B4) Windows 2000/XP: Start Mail key +const int VK_MEDIA_LAUNCH_MAIL = 0xB4; + +// VK_LAUNCH_MEDIA_SELECT (B5) Windows 2000/XP: Select Media key +const int VK_MEDIA_LAUNCH_MEDIA_SELECT = 0xB5; + +// VK_LAUNCH_APP1 (B6) Windows 2000/XP: Start Application 1 key +const int VK_MEDIA_LAUNCH_APP1 = 0xB6; + +// VK_LAUNCH_APP2 (B7) Windows 2000/XP: Start Application 2 key +const int VK_MEDIA_LAUNCH_APP2 = 0xB7; + +// VK_OEM_1 (BA) Used for miscellaneous characters; it can vary by keyboard. +// Windows 2000/XP: For the US standard keyboard, the ';:' key +const int VK_OEM_1 = 0xBA; + +// VK_OEM_PLUS (BB) Windows 2000/XP: For any country/region, the '+' key +const int VK_OEM_PLUS = 0xBB; + +// VK_OEM_COMMA (BC) Windows 2000/XP: For any country/region, the ',' key +const int VK_OEM_COMMA = 0xBC; + +// VK_OEM_MINUS (BD) Windows 2000/XP: For any country/region, the '-' key +const int VK_OEM_MINUS = 0xBD; + +// VK_OEM_PERIOD (BE) Windows 2000/XP: For any country/region, the '.' key +const int VK_OEM_PERIOD = 0xBE; + +// VK_OEM_2 (BF) Used for miscellaneous characters; it can vary by keyboard. +// Windows 2000/XP: For the US standard keyboard, the '/?' key +const int VK_OEM_2 = 0xBF; + +// VK_OEM_3 (C0) Used for miscellaneous characters; it can vary by keyboard. +// Windows 2000/XP: For the US standard keyboard, the '`~' key +const int VK_OEM_3 = 0xC0; + +// VK_OEM_4 (DB) Used for miscellaneous characters; it can vary by keyboard. +// Windows 2000/XP: For the US standard keyboard, the '[{' key +const int VK_OEM_4 = 0xDB; + +// VK_OEM_5 (DC) Used for miscellaneous characters; it can vary by keyboard. +// Windows 2000/XP: For the US standard keyboard, the '\|' key +const int VK_OEM_5 = 0xDC; + +// VK_OEM_6 (DD) Used for miscellaneous characters; it can vary by keyboard. +// Windows 2000/XP: For the US standard keyboard, the ']}' key +const int VK_OEM_6 = 0xDD; + +// VK_OEM_7 (DE) Used for miscellaneous characters; it can vary by keyboard. +// Windows 2000/XP: For the US standard keyboard, the +// 'single-quote/double-quote' key +const int VK_OEM_7 = 0xDE; + +// VK_OEM_8 (DF) Used for miscellaneous characters; it can vary by keyboard. +const int VK_OEM_8 = 0xDF; + +// VK_OEM_102 (E2) Windows 2000/XP: Either the angle bracket key or the +// backslash key on the RT 102-key keyboard +const int VK_OEM_102 = 0xE2; + +// VK_PROCESSKEY (E5) Windows 95/98/Me, Windows NT 4.0, Windows 2000/XP: IME +// PROCESS key +const int VK_PROCESSKEY = 0xE5; + +// VK_PACKET (E7) Windows 2000/XP: Used to pass Unicode characters as if they +// were keystrokes. The VK_PACKET key is the low word of a 32-bit Virtual Key +// value used for non-keyboard input methods. For more information, see Remark +// in KEYBDINPUT,SendInput, WM_KEYDOWN, and WM_KEYUP +const int VK_PACKET = 0xE7; + +// VK_ATTN (F6) Attn key +const int VK_ATTN = 0xF6; + +// VK_CRSEL (F7) CrSel key +const int VK_CRSEL = 0xF7; + +// VK_EXSEL (F8) ExSel key +const int VK_EXSEL = 0xF8; + +// VK_EREOF (F9) Erase EOF key +const int VK_EREOF = 0xF9; + +// VK_PLAY (FA) Play key +const int VK_PLAY = 0xFA; + +// VK_ZOOM (FB) Zoom key +const int VK_ZOOM = 0xFB; + +// VK_NONAME (FC) Reserved for future use +const int VK_NONAME = 0xFC; + +// VK_PA1 (FD) PA1 key +const int VK_PA1 = 0xFD; + +// VK_OEM_CLEAR (FE) Clear key +const int VK_OEM_CLEAR = 0xFE; + +const int VK_UNKNOWN = 0; +} // namespace WebCore + +#endif // LKCEF_KEYBOARD_CODES_H