From 4e96eb77e0755edf5a4351d61fa36d60415f1db6 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Mon, 11 Sep 2023 13:58:58 +0200 Subject: [PATCH] perf: Use `ImageWriter` instead of OpenGL Pipeline for faster processing (#1789) * perf: Use `ImageWriter` instead of OpenGL Pipeline for faster processing * chore: Remove C++ part of OpenGL pipeline * Clean up * Update README.md --- README.md | 2 +- package/android/CMakeLists.txt | 6 - .../android/src/main/cpp/OpenGLContext.cpp | 163 ------------------ package/android/src/main/cpp/OpenGLContext.h | 73 -------- package/android/src/main/cpp/OpenGLError.h | 34 ---- .../android/src/main/cpp/OpenGLRenderer.cpp | 74 -------- package/android/src/main/cpp/OpenGLRenderer.h | 61 ------- package/android/src/main/cpp/OpenGLTexture.h | 22 --- .../src/main/cpp/PassThroughShader.cpp | 111 ------------ .../android/src/main/cpp/PassThroughShader.h | 84 --------- .../android/src/main/cpp/VideoPipeline.cpp | 119 ------------- package/android/src/main/cpp/VideoPipeline.h | 66 ------- package/android/src/main/cpp/VisionCamera.cpp | 2 - .../main/java/com/mrousavy/camera/Errors.kt | 21 +-- .../com/mrousavy/camera/core/VideoPipeline.kt | 141 ++++----------- 15 files changed, 36 insertions(+), 943 deletions(-) delete mode 100644 package/android/src/main/cpp/OpenGLContext.cpp delete mode 100644 package/android/src/main/cpp/OpenGLContext.h delete mode 100644 package/android/src/main/cpp/OpenGLError.h delete mode 100644 package/android/src/main/cpp/OpenGLRenderer.cpp delete mode 100644 package/android/src/main/cpp/OpenGLRenderer.h delete mode 100644 package/android/src/main/cpp/OpenGLTexture.h delete mode 100644 package/android/src/main/cpp/PassThroughShader.cpp delete mode 100644 package/android/src/main/cpp/PassThroughShader.h delete mode 100644 package/android/src/main/cpp/VideoPipeline.cpp delete mode 100644 package/android/src/main/cpp/VideoPipeline.h diff --git a/README.md b/README.md index e6acb2da0c..8348650ddd 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ VisionCamera is a powerful and fast Camera component for React Native. It featur * 🔍 Smooth zooming (Reanimated) * ⏯️ Fast pause and resume * 🌓 HDR & Night modes -* ⚡ Custom C++/GPU accelerated video pipeline (OpenGL) +* ⚡ Highly efficient C++/GPU buffers Install VisionCamera from npm: diff --git a/package/android/CMakeLists.txt b/package/android/CMakeLists.txt index bf660ce279..09c335ab8f 100644 --- a/package/android/CMakeLists.txt +++ b/package/android/CMakeLists.txt @@ -21,10 +21,6 @@ add_library( SHARED ../cpp/JSITypedArray.cpp src/main/cpp/VisionCamera.cpp - src/main/cpp/VideoPipeline.cpp - src/main/cpp/PassThroughShader.cpp - src/main/cpp/OpenGLContext.cpp - src/main/cpp/OpenGLRenderer.cpp # Frame Processor src/main/cpp/frameprocessor/FrameHostObject.cpp src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp @@ -58,8 +54,6 @@ target_link_libraries( ReactAndroid::jsi # <-- RN: JSI ReactAndroid::reactnativejni # <-- RN: React Native JNI bindings fbjni::fbjni # <-- fbjni - GLESv2 # <-- OpenGL (for VideoPipeline) - EGL # <-- OpenGL (EGL) (for VideoPipeline) ) # Optionally also add Frame Processors here diff --git a/package/android/src/main/cpp/OpenGLContext.cpp b/package/android/src/main/cpp/OpenGLContext.cpp deleted file mode 100644 index 307754cb53..0000000000 --- a/package/android/src/main/cpp/OpenGLContext.cpp +++ /dev/null @@ -1,163 +0,0 @@ -// -// Created by Marc Rousavy on 29.08.23. -// - -#include "OpenGLContext.h" - -#include -#include -#include - -#include -#include - -#include "OpenGLError.h" - -namespace vision { - -std::shared_ptr OpenGLContext::CreateWithOffscreenSurface() { - return std::unique_ptr(new OpenGLContext()); -} - -OpenGLContext::~OpenGLContext() { - destroy(); -} - -void OpenGLContext::destroy() { - if (display != EGL_NO_DISPLAY) { - eglMakeCurrent(display, offscreenSurface, offscreenSurface, context); - if (offscreenSurface != EGL_NO_SURFACE) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Surface..."); - eglDestroySurface(display, offscreenSurface); - offscreenSurface = EGL_NO_SURFACE; - } - if (context != EGL_NO_CONTEXT) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Context..."); - eglDestroyContext(display, context); - context = EGL_NO_CONTEXT; - } - __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Display..."); - eglTerminate(display); - display = EGL_NO_DISPLAY; - config = nullptr; - } -} - -void OpenGLContext::ensureOpenGL() { - bool successful; - // EGLDisplay - if (display == EGL_NO_DISPLAY) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLDisplay.."); - display = eglGetDisplay(EGL_DEFAULT_DISPLAY); - if (display == EGL_NO_DISPLAY) - throw OpenGLError("Failed to get default OpenGL Display!"); - - EGLint major; - EGLint minor; - successful = eglInitialize(display, &major, &minor); - if (!successful) - throw OpenGLError("Failed to initialize OpenGL!"); - } - - // EGLConfig - if (config == nullptr) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLConfig.."); - EGLint attributes[] = {EGL_RENDERABLE_TYPE, - EGL_OPENGL_ES2_BIT, - EGL_SURFACE_TYPE, - EGL_WINDOW_BIT, - EGL_RED_SIZE, - 8, - EGL_GREEN_SIZE, - 8, - EGL_BLUE_SIZE, - 8, - EGL_ALPHA_SIZE, - 8, - EGL_DEPTH_SIZE, - 0, - EGL_STENCIL_SIZE, - 0, - EGL_NONE}; - EGLint numConfigs; - successful = eglChooseConfig(display, attributes, &config, 1, &numConfigs); - if (!successful || numConfigs == 0) - throw OpenGLError("Failed to choose OpenGL config!"); - } - - // EGLContext - if (context == EGL_NO_CONTEXT) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLContext.."); - EGLint contextAttributes[] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE}; - context = eglCreateContext(display, config, nullptr, contextAttributes); - if (context == EGL_NO_CONTEXT) - throw OpenGLError("Failed to create OpenGL context!"); - } - - // EGLSurface - if (offscreenSurface == EGL_NO_SURFACE) { - // If we don't have a surface at all - __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing 1x1 offscreen pbuffer EGLSurface.."); - EGLint attributes[] = {EGL_WIDTH, 1, EGL_HEIGHT, 1, EGL_NONE}; - offscreenSurface = eglCreatePbufferSurface(display, config, attributes); - if (offscreenSurface == EGL_NO_SURFACE) - throw OpenGLError("Failed to create OpenGL Surface!"); - } -} - -void OpenGLContext::use() { - this->use(offscreenSurface); -} - -void OpenGLContext::use(EGLSurface surface) { - if (surface == EGL_NO_SURFACE) - throw OpenGLError("Cannot render to a null Surface!"); - - // 1. Make sure the OpenGL context is initialized - this->ensureOpenGL(); - - // 2. Make the OpenGL context current - bool successful = eglMakeCurrent(display, surface, surface, context); - if (!successful || eglGetError() != EGL_SUCCESS) - throw OpenGLError("Failed to use current OpenGL context!"); - - // 3. Caller can now render to this surface -} - -void OpenGLContext::flush() const { - bool successful = eglSwapBuffers(display, eglGetCurrentSurface(EGL_DRAW)); - if (!successful || eglGetError() != EGL_SUCCESS) - throw OpenGLError("Failed to swap OpenGL buffers!"); -} - -OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width, int height) { - // 1. Make sure the OpenGL context is initialized - this->ensureOpenGL(); - - // 2. Make the OpenGL context current - bool successful = eglMakeCurrent(display, offscreenSurface, offscreenSurface, context); - if (!successful || eglGetError() != EGL_SUCCESS) - throw OpenGLError("Failed to use current OpenGL context!"); - - GLuint textureId; - glGenTextures(1, &textureId); - - GLenum target; - switch (type) { - case OpenGLTexture::Type::ExternalOES: - target = GL_TEXTURE_EXTERNAL_OES; - break; - case OpenGLTexture::Type::Texture2D: - target = GL_TEXTURE_2D; - break; - default: - throw std::runtime_error("Invalid OpenGL Texture Type!"); - } - glBindTexture(target, textureId); - glTexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - - return {.id = textureId, .target = target, .width = width, .height = height}; -} - -} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLContext.h b/package/android/src/main/cpp/OpenGLContext.h deleted file mode 100644 index 886bf57a7f..0000000000 --- a/package/android/src/main/cpp/OpenGLContext.h +++ /dev/null @@ -1,73 +0,0 @@ -// -// Created by Marc Rousavy on 29.08.23. -// - -#pragma once - -#include -#include - -#include -#include - -#include "OpenGLTexture.h" -#include "PassThroughShader.h" - -namespace vision { - -/** - * An OpenGL Context that can be used to render to different surfaces. - * By default, it creates an off-screen PixelBuffer surface. - */ -class OpenGLContext { -public: - /** - * Create a new instance of the OpenGLContext that draws to an off-screen PixelBuffer surface. - * This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread. - */ - static std::shared_ptr CreateWithOffscreenSurface(); - /** - * Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called. - */ - ~OpenGLContext(); - - /** - * Use this OpenGL Context to render to the given EGLSurface. - * After the `renderFunc` returns, the default offscreen PixelBuffer surface becomes active again. - */ - void use(EGLSurface surface); - - /** - * Use this OpenGL Context to render to the offscreen PixelBuffer surface. - */ - void use(); - - /** - * Flushes all drawing operations by swapping the buffers and submitting the Frame to the GPU - */ - void flush() const; - - /** - * Create a new texture on this context - */ - OpenGLTexture createTexture(OpenGLTexture::Type type, int width, int height); - -public: - EGLDisplay display = EGL_NO_DISPLAY; - EGLContext context = EGL_NO_CONTEXT; - EGLSurface offscreenSurface = EGL_NO_SURFACE; - EGLConfig config = nullptr; - -private: - OpenGLContext() = default; - void destroy(); - void ensureOpenGL(); - -private: - PassThroughShader _passThroughShader; - -private: - static constexpr auto TAG = "OpenGLContext"; -}; - -} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLError.h b/package/android/src/main/cpp/OpenGLError.h deleted file mode 100644 index 72e97d0dc4..0000000000 --- a/package/android/src/main/cpp/OpenGLError.h +++ /dev/null @@ -1,34 +0,0 @@ -// -// Created by Marc Rousavy on 09.08.23. -// - -#pragma once - -#include -#include -#include - -namespace vision { - -inline std::string getEglErrorIfAny() { - EGLint error = glGetError(); - if (error != GL_NO_ERROR) - return " Error: " + std::to_string(error); - error = eglGetError(); - if (error != EGL_SUCCESS) - return " Error: " + std::to_string(error); - return ""; -} - -class OpenGLError : public std::runtime_error { -public: - explicit OpenGLError(const std::string&& message) : std::runtime_error(message + getEglErrorIfAny()) {} - - static inline void checkIfError(const std::string&& message) { - auto error = getEglErrorIfAny(); - if (error.length() > 0) - throw std::runtime_error(message + error); - } -}; - -} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLRenderer.cpp b/package/android/src/main/cpp/OpenGLRenderer.cpp deleted file mode 100644 index 3768399f64..0000000000 --- a/package/android/src/main/cpp/OpenGLRenderer.cpp +++ /dev/null @@ -1,74 +0,0 @@ -// -// Created by Marc Rousavy on 29.08.23. -// - -#include "OpenGLRenderer.h" - -#include -#include -#include - -#include -#include - -#include - -#include "OpenGLError.h" - -namespace vision { - -std::unique_ptr OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface) { - return std::unique_ptr(new OpenGLRenderer(std::move(context), surface)); -} - -OpenGLRenderer::OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface) { - _context = std::move(context); - _outputSurface = surface; - _width = ANativeWindow_getWidth(surface); - _height = ANativeWindow_getHeight(surface); -} - -OpenGLRenderer::~OpenGLRenderer() { - if (_outputSurface != nullptr) { - ANativeWindow_release(_outputSurface); - } - destroy(); -} - -void OpenGLRenderer::destroy() { - if (_context != nullptr && _surface != EGL_NO_DISPLAY) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Surface..."); - eglDestroySurface(_context->display, _surface); - _surface = EGL_NO_SURFACE; - } -} - -void OpenGLRenderer::renderTextureToSurface(const OpenGLTexture& texture, float* transformMatrix) { - if (_surface == EGL_NO_SURFACE) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Creating Window Surface..."); - _context->use(); - _surface = eglCreateWindowSurface(_context->display, _context->config, _outputSurface, nullptr); - } - - // 1. Activate the OpenGL context for this surface - _context->use(_surface); - - // 2. Set the viewport for rendering - glViewport(0, 0, _width, _height); - glDisable(GL_BLEND); - - // 3. Bind the input texture - glBindTexture(texture.target, texture.id); - glTexParameteri(texture.target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(texture.target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(texture.target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(texture.target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - - // 4. Draw it using the pass-through shader which also applies transforms - _passThroughShader.draw(texture, transformMatrix); - - // 5. Swap buffers to pass it to the window surface - eglSwapBuffers(_context->display, _surface); -} - -} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLRenderer.h b/package/android/src/main/cpp/OpenGLRenderer.h deleted file mode 100644 index b2b96cb44e..0000000000 --- a/package/android/src/main/cpp/OpenGLRenderer.h +++ /dev/null @@ -1,61 +0,0 @@ -// -// Created by Marc Rousavy on 29.08.23. -// - -#pragma once - -#include "PassThroughShader.h" -#include -#include -#include -#include - -#include "OpenGLContext.h" -#include "OpenGLTexture.h" - -namespace vision { - -class OpenGLRenderer { -public: - /** - * Create a new instance of the OpenGLRenderer that draws to an on-screen window surface. - * This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread. - * - * Note: The `surface` is considered moved, and the OpenGL context will release it when it is - * being deleted. - */ - static std::unique_ptr CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface); - /** - * Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called. - */ - ~OpenGLRenderer(); - - /** - * Renders the given Texture to the Surface - */ - void renderTextureToSurface(const OpenGLTexture& texture, float* transformMatrix); - - /** - * Destroys the OpenGL context. This needs to be called on the same thread that `use()` was - * called. After calling `destroy()`, it is legal to call `use()` again, which will re-construct - * everything. - */ - void destroy(); - -private: - explicit OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface); - -private: - int _width = 0, _height = 0; - std::shared_ptr _context; - ANativeWindow* _outputSurface; - EGLSurface _surface = EGL_NO_SURFACE; - -private: - PassThroughShader _passThroughShader; - -private: - static constexpr auto TAG = "OpenGLRenderer"; -}; - -} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLTexture.h b/package/android/src/main/cpp/OpenGLTexture.h deleted file mode 100644 index ec93c2abe4..0000000000 --- a/package/android/src/main/cpp/OpenGLTexture.h +++ /dev/null @@ -1,22 +0,0 @@ -// -// Created by Marc Rousavy on 30.08.23. -// - -#pragma once - -#include -#include -#include - -struct OpenGLTexture { - enum Type { Texture2D, ExternalOES }; - - // The ID of the texture as returned in glGenTextures(..) - GLuint id; - // GL_TEXTURE_2D or GL_TEXTURE_EXTERNAL_OES - GLenum target; - - // Width and height of the texture - int width = 0; - int height = 0; -}; diff --git a/package/android/src/main/cpp/PassThroughShader.cpp b/package/android/src/main/cpp/PassThroughShader.cpp deleted file mode 100644 index ad51fdf720..0000000000 --- a/package/android/src/main/cpp/PassThroughShader.cpp +++ /dev/null @@ -1,111 +0,0 @@ -// -// Created by Marc Rousavy on 28.08.23. -// - -#include "PassThroughShader.h" -#include "OpenGLError.h" -#include -#include -#include -#include -#include - -namespace vision { - -PassThroughShader::~PassThroughShader() { - if (_programId != NO_SHADER) { - glDeleteProgram(_programId); - _programId = NO_SHADER; - } - - if (_vertexBuffer != NO_BUFFER) { - glDeleteBuffers(1, &_vertexBuffer); - _vertexBuffer = NO_BUFFER; - } -} - -void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix) { - // 1. Set up Shader Program - if (_programId == NO_SHADER) { - _programId = createProgram(); - glUseProgram(_programId); - _vertexParameters = { - .aPosition = glGetAttribLocation(_programId, "aPosition"), - .aTexCoord = glGetAttribLocation(_programId, "aTexCoord"), - .uTransformMatrix = glGetUniformLocation(_programId, "uTransformMatrix"), - }; - _fragmentParameters = { - .uTexture = glGetUniformLocation(_programId, "uTexture"), - }; - } - - glUseProgram(_programId); - - // 2. Set up Vertices Buffer - if (_vertexBuffer == NO_BUFFER) { - glGenBuffers(1, &_vertexBuffer); - glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); - glBufferData(GL_ARRAY_BUFFER, sizeof(VERTICES), VERTICES, GL_STATIC_DRAW); - } - - // 3. Pass all uniforms/attributes for vertex shader - glEnableVertexAttribArray(_vertexParameters.aPosition); - glVertexAttribPointer(_vertexParameters.aPosition, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex), - reinterpret_cast(offsetof(Vertex, position))); - - glEnableVertexAttribArray(_vertexParameters.aTexCoord); - glVertexAttribPointer(_vertexParameters.aTexCoord, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex), - reinterpret_cast(offsetof(Vertex, texCoord))); - - glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, transformMatrix); - - // 4. Pass texture to fragment shader - glActiveTexture(GL_TEXTURE0); - glBindTexture(texture.target, texture.id); - glUniform1i(_fragmentParameters.uTexture, 0); - - // 5. Draw! - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); -} - -GLuint PassThroughShader::loadShader(GLenum shaderType, const char* shaderCode) { - GLuint shader = glCreateShader(shaderType); - if (shader == 0) - throw OpenGLError("Failed to load shader!"); - - glShaderSource(shader, 1, &shaderCode, nullptr); - glCompileShader(shader); - GLint compileStatus = GL_FALSE; - glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); - if (compileStatus == GL_FALSE) { - glDeleteShader(shader); - throw OpenGLError("Failed to compile shader!"); - } - return shader; -} - -GLuint PassThroughShader::createProgram() { - GLuint vertexShader = loadShader(GL_VERTEX_SHADER, VERTEX_SHADER); - GLuint fragmentShader = loadShader(GL_FRAGMENT_SHADER, FRAGMENT_SHADER); - - GLuint program = glCreateProgram(); - if (program == 0) - throw OpenGLError("Failed to create pass-through program!"); - - glAttachShader(program, vertexShader); - OpenGLError::checkIfError("Failed to attach Vertex Shader!"); - - glAttachShader(program, fragmentShader); - OpenGLError::checkIfError("Failed to attach Fragment Shader!"); - - glLinkProgram(program); - GLint linkStatus = GL_FALSE; - glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); - if (!linkStatus) { - glDeleteProgram(program); - throw OpenGLError("Failed to load pass-through program!"); - } - return program; -} - -} // namespace vision diff --git a/package/android/src/main/cpp/PassThroughShader.h b/package/android/src/main/cpp/PassThroughShader.h deleted file mode 100644 index b199b3dacc..0000000000 --- a/package/android/src/main/cpp/PassThroughShader.h +++ /dev/null @@ -1,84 +0,0 @@ -// -// Created by Marc Rousavy on 28.08.23. -// - -#pragma once - -#include -#include - -#include "OpenGLTexture.h" - -namespace vision { - -#define NO_SHADER 0 -#define NO_POSITION 0 -#define NO_BUFFER 0 - -struct Vertex { - GLfloat position[2]; - GLfloat texCoord[2]; -}; - -class PassThroughShader { -public: - PassThroughShader() = default; - ~PassThroughShader(); - - /** - * Draw the texture using this shader. - * Note: At the moment, only EXTERNAL textures are supported by the Shader. - */ - void draw(const OpenGLTexture& texture, float* transformMatrix); - -private: - // Loading - static GLuint loadShader(GLenum shaderType, const char* shaderCode); - static GLuint createProgram(); - -private: - // Parameters - GLuint _programId = NO_SHADER; - GLuint _vertexBuffer = NO_BUFFER; - struct VertexParameters { - GLint aPosition = NO_POSITION; - GLint aTexCoord = NO_POSITION; - GLint uTransformMatrix = NO_POSITION; - } _vertexParameters; - struct FragmentParameters { - GLint uTexture = NO_POSITION; - } _fragmentParameters; - -private: - // Statics - static constexpr Vertex VERTICES[] = { - {{-1.0f, -1.0f}, {0.0f, 0.0f}}, // bottom-left - {{1.0f, -1.0f}, {1.0f, 0.0f}}, // bottom-right - {{-1.0f, 1.0f}, {0.0f, 1.0f}}, // top-left - {{1.0f, 1.0f}, {1.0f, 1.0f}} // top-right - }; - - static constexpr char VERTEX_SHADER[] = R"( - attribute vec4 aPosition; - attribute vec2 aTexCoord; - uniform mat4 uTransformMatrix; - varying vec2 vTexCoord; - - void main() { - gl_Position = aPosition; - vTexCoord = (uTransformMatrix * vec4(aTexCoord, 0.0, 1.0)).xy; - } - )"; - static constexpr char FRAGMENT_SHADER[] = R"( - #extension GL_OES_EGL_image_external : require - precision mediump float; - varying vec2 vTexCoord; - uniform samplerExternalOES uTexture; - - void main() { - gl_FragColor = texture2D(uTexture, vTexCoord); - } - )"; -}; - -} // namespace vision diff --git a/package/android/src/main/cpp/VideoPipeline.cpp b/package/android/src/main/cpp/VideoPipeline.cpp deleted file mode 100644 index 387e2fcc1e..0000000000 --- a/package/android/src/main/cpp/VideoPipeline.cpp +++ /dev/null @@ -1,119 +0,0 @@ -// -// Created by Marc Rousavy on 25.08.23. -// - -#include "VideoPipeline.h" -#include "OpenGLError.h" - -#include -#include -#include -#include -#include -#include - -#include - -#include "JFrameProcessor.h" -#include "OpenGLTexture.h" - -namespace vision { - -jni::local_ref VideoPipeline::initHybrid(jni::alias_ref jThis, int width, int height) { - return makeCxxInstance(jThis, width, height); -} - -VideoPipeline::VideoPipeline(jni::alias_ref jThis, int width, int height) : _javaPart(jni::make_global(jThis)) { - _width = width; - _height = height; - _context = OpenGLContext::CreateWithOffscreenSurface(); -} - -VideoPipeline::~VideoPipeline() { - // 1. Remove output surfaces - removeFrameProcessorOutputSurface(); - removeRecordingSessionOutputSurface(); - // 2. Delete the input textures - if (_inputTexture != std::nullopt) { - glDeleteTextures(1, &_inputTexture->id); - _inputTexture = std::nullopt; - } - // 3. Destroy the OpenGL context - _context = nullptr; -} - -void VideoPipeline::removeFrameProcessorOutputSurface() { - if (_frameProcessorOutput) - _frameProcessorOutput->destroy(); - _frameProcessorOutput = nullptr; -} - -void VideoPipeline::setFrameProcessorOutputSurface(jobject surface) { - // 1. Delete existing output surface - removeFrameProcessorOutputSurface(); - - // 2. Set new output surface if it is not null - ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); - _frameProcessorOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window); -} - -void VideoPipeline::removeRecordingSessionOutputSurface() { - if (_recordingSessionOutput) - _recordingSessionOutput->destroy(); - _recordingSessionOutput = nullptr; -} - -void VideoPipeline::setRecordingSessionOutputSurface(jobject surface) { - // 1. Delete existing output surface - removeRecordingSessionOutputSurface(); - - // 2. Set new output surface if it is not null - ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); - _recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window); -} - -int VideoPipeline::getInputTextureId() { - if (_inputTexture == std::nullopt) { - _inputTexture = _context->createTexture(OpenGLTexture::Type::ExternalOES, _width, _height); - } - - return static_cast(_inputTexture->id); -} - -void VideoPipeline::onBeforeFrame() { - _context->use(); - - glBindTexture(_inputTexture->target, _inputTexture->id); -} - -void VideoPipeline::onFrame(jni::alias_ref transformMatrixParam) { - // Get the OpenGL transform Matrix (transforms, scales, rotations) - float transformMatrix[16]; - transformMatrixParam->getRegion(0, 16, transformMatrix); - - OpenGLTexture& texture = _inputTexture.value(); - - if (_frameProcessorOutput) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to FrameProcessor.."); - _frameProcessorOutput->renderTextureToSurface(texture, transformMatrix); - } - if (_recordingSessionOutput) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession.."); - _recordingSessionOutput->renderTextureToSurface(texture, transformMatrix); - } -} - -void VideoPipeline::registerNatives() { - registerHybrid({ - makeNativeMethod("initHybrid", VideoPipeline::initHybrid), - makeNativeMethod("setFrameProcessorOutputSurface", VideoPipeline::setFrameProcessorOutputSurface), - makeNativeMethod("removeFrameProcessorOutputSurface", VideoPipeline::removeFrameProcessorOutputSurface), - makeNativeMethod("setRecordingSessionOutputSurface", VideoPipeline::setRecordingSessionOutputSurface), - makeNativeMethod("removeRecordingSessionOutputSurface", VideoPipeline::removeRecordingSessionOutputSurface), - makeNativeMethod("getInputTextureId", VideoPipeline::getInputTextureId), - makeNativeMethod("onBeforeFrame", VideoPipeline::onBeforeFrame), - makeNativeMethod("onFrame", VideoPipeline::onFrame), - }); -} - -} // namespace vision diff --git a/package/android/src/main/cpp/VideoPipeline.h b/package/android/src/main/cpp/VideoPipeline.h deleted file mode 100644 index ca2551f137..0000000000 --- a/package/android/src/main/cpp/VideoPipeline.h +++ /dev/null @@ -1,66 +0,0 @@ -// -// Created by Marc Rousavy on 25.08.23. -// - -#pragma once - -#include "OpenGLContext.h" -#include "OpenGLRenderer.h" -#include "PassThroughShader.h" -#include -#include -#include -#include -#include -#include - -namespace vision { - -using namespace facebook; - -class VideoPipeline : public jni::HybridClass { -public: - static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/core/VideoPipeline;"; - static jni::local_ref initHybrid(jni::alias_ref jThis, int width, int height); - static void registerNatives(); - -public: - ~VideoPipeline(); - - // -> SurfaceTexture input - int getInputTextureId(); - - // <- Frame Processor output - void setFrameProcessorOutputSurface(jobject surface); - void removeFrameProcessorOutputSurface(); - - // <- MediaRecorder output - void setRecordingSessionOutputSurface(jobject surface); - void removeRecordingSessionOutputSurface(); - - // Frame callbacks - void onBeforeFrame(); - void onFrame(jni::alias_ref transformMatrix); - -private: - // Private constructor. Use `create(..)` to create new instances. - explicit VideoPipeline(jni::alias_ref jThis, int width, int height); - -private: - // Input Surface Texture - std::optional _inputTexture = std::nullopt; - int _width = 0; - int _height = 0; - - // Output Contexts - std::shared_ptr _context = nullptr; - std::unique_ptr _frameProcessorOutput = nullptr; - std::unique_ptr _recordingSessionOutput = nullptr; - -private: - friend HybridBase; - jni::global_ref _javaPart; - static constexpr auto TAG = "VideoPipeline"; -}; - -} // namespace vision diff --git a/package/android/src/main/cpp/VisionCamera.cpp b/package/android/src/main/cpp/VisionCamera.cpp index 3637e7e98a..bab71f7dac 100644 --- a/package/android/src/main/cpp/VisionCamera.cpp +++ b/package/android/src/main/cpp/VisionCamera.cpp @@ -1,7 +1,6 @@ #include "JFrameProcessor.h" #include "JVisionCameraProxy.h" #include "JVisionCameraScheduler.h" -#include "VideoPipeline.h" #include "VisionCameraProxy.h" #include #include @@ -11,7 +10,6 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) { vision::VisionCameraInstaller::registerNatives(); vision::JVisionCameraProxy::registerNatives(); vision::JVisionCameraScheduler::registerNatives(); - vision::VideoPipeline::registerNatives(); #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS vision::JFrameProcessor::registerNatives(); #endif diff --git a/package/android/src/main/java/com/mrousavy/camera/Errors.kt b/package/android/src/main/java/com/mrousavy/camera/Errors.kt index 9433a4d38c..f9e6664bc2 100644 --- a/package/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/package/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -4,27 +4,12 @@ import com.mrousavy.camera.parsers.CameraDeviceError import com.mrousavy.camera.core.outputs.CameraOutputs abstract class CameraError( - /** - * The domain of the error. Error domains are used to group errors. - * - * Example: "permission" - */ + // example: "permission" val domain: String, - /** - * The id of the error. Errors are uniquely identified under a given domain. - * - * Example: "microphone-permission-denied" - */ + // example: "microphone-permission-denied" val id: String, - /** - * A detailed error description of "what went wrong". - * - * Example: "The microphone permission was denied!" - */ + // example: "The microphone permission was denied!" message: String, - /** - * A throwable that caused this error. - */ cause: Throwable? = null ) : Throwable("[$domain/$id] $message", cause) diff --git a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt index 42b1cb7515..1da47504c2 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt @@ -1,163 +1,86 @@ package com.mrousavy.camera.core import android.graphics.ImageFormat -import android.graphics.SurfaceTexture import android.media.ImageReader import android.media.ImageWriter import android.media.MediaRecorder import android.util.Log import android.view.Surface -import com.facebook.jni.HybridData +import com.mrousavy.camera.CameraQueues import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor import com.mrousavy.camera.parsers.Orientation import java.io.Closeable -/** - * An OpenGL pipeline for streaming Camera Frames to one or more outputs. - * Currently, [VideoPipeline] can stream to a [FrameProcessor] and a [MediaRecorder]. - * - * @param [width] The width of the Frames to stream (> 0) - * @param [height] The height of the Frames to stream (> 0) - * @param [format] The format of the Frames to stream. ([ImageFormat.PRIVATE], [ImageFormat.YUV_420_888] or [ImageFormat.JPEG]) - */ -@Suppress("KotlinJniMissingFunction") +@Suppress("JoinDeclarationAndAssignment") class VideoPipeline(val width: Int, val height: Int, val format: Int = ImageFormat.PRIVATE, - private val isMirrored: Boolean = false): SurfaceTexture.OnFrameAvailableListener, Closeable { + private val isMirrored: Boolean = false): ImageReader.OnImageAvailableListener, Closeable { companion object { private const val MAX_IMAGES = 3 private const val TAG = "VideoPipeline" } - private val mHybridData: HybridData - private var openGLTextureId: Int? = null - private var transformMatrix = FloatArray(16) - private var isActive = true - // Output 1 private var frameProcessor: FrameProcessor? = null - private var imageReader: ImageReader? = null // Output 2 private var recordingSession: RecordingSession? = null + private var recordingSessionImageWriter: ImageWriter? = null // Input - private val surfaceTexture: SurfaceTexture + private val imageReader: ImageReader val surface: Surface init { - mHybridData = initHybrid(width, height) - surfaceTexture = SurfaceTexture(false) - surfaceTexture.setDefaultBufferSize(width, height) - surfaceTexture.setOnFrameAvailableListener(this) - surface = Surface(surfaceTexture) + imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES) + imageReader.setOnImageAvailableListener(this, CameraQueues.videoQueue.handler) + surface = imageReader.surface } override fun close() { synchronized(this) { - isActive = false - imageReader?.close() - imageReader = null + imageReader.close() frameProcessor = null + recordingSessionImageWriter?.close() + recordingSessionImageWriter = null recordingSession = null - surfaceTexture.release() - mHybridData.resetNative() } } - override fun onFrameAvailable(surfaceTexture: SurfaceTexture) { + fun setFrameProcessorOutput(frameProcessor: FrameProcessor?) { + this.frameProcessor = frameProcessor + } + + fun setRecordingSessionOutput(recordingSession: RecordingSession?) { synchronized(this) { - if (!isActive) return@synchronized + this.recordingSessionImageWriter?.close() + this.recordingSessionImageWriter = null + this.recordingSession = recordingSession - // 1. Attach Surface to OpenGL context - if (openGLTextureId == null) { - openGLTextureId = getInputTextureId() - surfaceTexture.attachToGLContext(openGLTextureId!!) - Log.i(TAG, "Attached Texture to Context $openGLTextureId") + if (recordingSession != null) { + this.recordingSessionImageWriter = ImageWriter.newInstance(recordingSession.surface, MAX_IMAGES) } - - // 2. Prepare the OpenGL context (eglMakeCurrent) - onBeforeFrame() - - // 3. Update the OpenGL texture - surfaceTexture.updateTexImage() - - // 4. Get the transform matrix from the SurfaceTexture (rotations/scales applied by Camera) - surfaceTexture.getTransformMatrix(transformMatrix) - - // 5. Draw it with applied rotation/mirroring - onFrame(transformMatrix) } } - private fun getImageReader(): ImageReader { - val imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES) - imageReader.setOnImageAvailableListener({ reader -> - Log.i("VideoPipeline", "ImageReader::onImageAvailable!") - val image = reader.acquireLatestImage() ?: return@setOnImageAvailableListener + override fun onImageAvailable(reader: ImageReader) { + val image = reader.acquireLatestImage() + if (image == null) { + Log.w(TAG, "ImageReader failed to acquire a new image!") + return + } - // TODO: Get correct orientation and isMirrored + // If we have a Frame Processor, call it + frameProcessor?.let { fp -> val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored) frame.incrementRefCount() - frameProcessor?.call(frame) + fp.call(frame) frame.decrementRefCount() - }, null) - return imageReader - } - - /** - * Configures the Pipeline to also call the given [FrameProcessor] (or null). - */ - fun setFrameProcessorOutput(frameProcessor: FrameProcessor?) { - synchronized(this) { - Log.i(TAG, "Setting $width x $height FrameProcessor Output...") - this.frameProcessor = frameProcessor - - if (frameProcessor != null) { - if (this.imageReader == null) { - // 1. Create new ImageReader that just calls the Frame Processor - this.imageReader = getImageReader() - } - - // 2. Configure OpenGL pipeline to stream Frames into the ImageReader's surface - setFrameProcessorOutputSurface(imageReader!!.surface) - } else { - // 1. Configure OpenGL pipeline to stop streaming Frames into the ImageReader's surface - removeFrameProcessorOutputSurface() - - // 2. Close the ImageReader - this.imageReader?.close() - this.imageReader = null - } } - } - /** - * Configures the Pipeline to also write Frames to a Surface from a [MediaRecorder] (or null) - */ - fun setRecordingSessionOutput(recordingSession: RecordingSession?) { - synchronized(this) { - Log.i(TAG, "Setting $width x $height RecordingSession Output...") - if (recordingSession != null) { - // Configure OpenGL pipeline to stream Frames into the Recording Session's surface - setRecordingSessionOutputSurface(recordingSession.surface) - this.recordingSession = recordingSession - } else { - // Configure OpenGL pipeline to stop streaming Frames into the Recording Session's surface - removeRecordingSessionOutputSurface() - this.recordingSession = null - } - } + // If we have a RecordingSession, pass the image through + recordingSessionImageWriter?.queueInputImage(image) } - - private external fun getInputTextureId(): Int - private external fun onBeforeFrame() - private external fun onFrame(transformMatrix: FloatArray) - private external fun setFrameProcessorOutputSurface(surface: Any) - private external fun removeFrameProcessorOutputSurface() - private external fun setRecordingSessionOutputSurface(surface: Any) - private external fun removeRecordingSessionOutputSurface() - private external fun initHybrid(width: Int, height: Int): HybridData }