fix: Fix Frame.orientation, it should always be fixed (#3077)

* fix: Fix `Frame.orientation`, it should always be fixed

* fix: Pass preview orientation

* Update CameraProps.ts

* Update useSkiaFrameProcessor.ts

* Update CameraSession.kt

docs/docs/guides/ORIENTATION.mdx

@@ -42,6 +42,8 @@ Frame Processors will stream frames in a potentially "wrong" orientation, and th
 :::info
 This needs to be handled manually, see [`Frame.orientation`](/docs/api/interfaces/Frame#orientation).
 For example, in MLKit just pass the `Frame`'s `orientation` to the `detect(...)` method.
+
+Instead of always rotating up-right to portrait, you might also want to rotate the Frame to either preview-, or output-orientation, depending on your use-case.
 :::
 
 ## Implementation
@@ -97,9 +99,12 @@ For a smoother user experience, you should animate changes to the UI rotation. U
 In a Frame Processor, frames are streamed in their native sensor orientation.
 This means even if the phone is rotated from portrait to landscape, the Frame's [`width`](/docs/api/interfaces/Frame#width) and [`height`](/docs/api/interfaces/Frame#height) stay the same.
 
-The Frame's [`orientation`](/docs/api/interfaces/Frame#orientation) represents it's orientation relative to the current target orientation.
+The Frame's [`orientation`](/docs/api/interfaces/Frame#orientation) represents the **image buffer's orientation, relative to the device's native portrait mode**.
+
+For example, if the Frame's `orientation` is `landscape-right`, it is 90° rotated and needs to be counter-rotated by -90° to appear "up-right".
+
+On an iPhone, "up-right" means portrait mode (the home-button is at the bottom). On an iPad, "up-right" might mean a landscape orientation.
 
-For example, if the phone is held in `portrait` mode and the Frame's `orientation` is `landscape-right`, it is 90° rotated and needs to be counter-rotated by -90° to appear "up-right".
 Instead of actually rotating pixels in the buffers, frame processor plugins just need to interpret the frame as being rotated.
 
 MLKit handles this via a `orientation` property on the `MLImage`/`VisionImage` object:
@@ -113,12 +118,25 @@ public override func callback(_ frame: Frame, withArguments _: [AnyHashable: Any
 }
 ```
 
+You can then either rotate to preview-, or output-orientation, depending on your use-case.
+
+#### Rotate `Frame.orientation` to output Orientation
+
+If you have a Frame Processor that detects objects or faces and the user holds the phone in a landscape orientation, your algorithm might not be able to detect the object or face because it is rotated.
+
+In this case you can just rotate the `Frame.orientation` by the `outputOrientation` (see [`onOutputOrientationChanged`](/docs/api/interfaces/CameraProps#onoutputorientationchanged)), and it will then be a landscape Frame if the user rotates the phone to landscape, or a portrait Frame if the user holds the phone in portrait.
+
+#### Rotate `Frame.orientation` to preview Orientation
+
+If you have a Frame Processor tht applies some drawing operations or provides visual feedback to the Preview, you don't want to use the `outputOrientation` as that can be different than the `previewOrientation`.
+
+In this case you can follow the same idea as above, just rotate the `Frame.orientation` by the `previewOrientation` (see [`onPreviewOrientationChanged`](/docs/api/interfaces/CameraProps#onprevieworientationchanged)) to receive a Frame in the same orientation the Preview view is currently in.
+
 ### Orientation in Skia Frame Processors
 
 A Skia Frame Processor applies orientation via rotation and translation. This means the coordinate system stays the same, but output will be rotated accordingly.
 For a `landscape-left` frame, `(0,0)` will not be top left, but rather top right.
 
-
 ## Mirroring (`isMirrored`)
 
 The photo-, video- and snapshot- outputs can be mirrored alongside the vertical axis (left/right flipped) by setting [`isMirrored`](/docs/api/interfaces/CameraProps#ismirrored) to `true`.

package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt

@@ -193,14 +193,14 @@ class CameraSession(internal val context: Context, internal val callback: Callba
     // Preview Orientation
     orientationManager.previewOrientation.toSurfaceRotation().let { previewRotation ->
       previewOutput?.targetRotation = previewRotation
+      codeScannerOutput?.targetRotation = previewRotation
     }
     // Outputs Orientation
     orientationManager.outputOrientation.toSurfaceRotation().let { outputRotation ->
       photoOutput?.targetRotation = outputRotation
       videoOutput?.targetRotation = outputRotation
-      frameProcessorOutput?.targetRotation = outputRotation
-      codeScannerOutput?.targetRotation = outputRotation
     }
+    // Frame Processor output will not receive a target rotation, user is responsible for rotating himself
   }
 
   interface Callback {

package/example/ios/Podfile.lock

@@ -1391,16 +1391,16 @@ PODS:
     - ReactCommon/turbomodule/core
     - Yoga
   - SocketRocket (0.7.0)
-  - VisionCamera (4.4.2):
-    - VisionCamera/Core (= 4.4.2)
-    - VisionCamera/FrameProcessors (= 4.4.2)
-    - VisionCamera/React (= 4.4.2)
-  - VisionCamera/Core (4.4.2)
-  - VisionCamera/FrameProcessors (4.4.2):
+  - VisionCamera (4.4.3):
+    - VisionCamera/Core (= 4.4.3)
+    - VisionCamera/FrameProcessors (= 4.4.3)
+    - VisionCamera/React (= 4.4.3)
+  - VisionCamera/Core (4.4.3)
+  - VisionCamera/FrameProcessors (4.4.3):
     - React
     - React-callinvoker
     - react-native-worklets-core
-  - VisionCamera/React (4.4.2):
+  - VisionCamera/React (4.4.3):
     - React-Core
     - VisionCamera/FrameProcessors
   - Yoga (0.0.0)
@@ -1688,7 +1688,7 @@ SPEC CHECKSUMS:
   RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8
   RNVectorIcons: 2a2f79274248390b80684ea3c4400bd374a15c90
   SocketRocket: abac6f5de4d4d62d24e11868d7a2f427e0ef940d
-  VisionCamera: 0292805e99ae8d121c8ec5bb8a9554f6f1c36b82
+  VisionCamera: 7435f20f7ee7756a1e307e986195a97764da5142
   Yoga: 2f71ecf38d934aecb366e686278102a51679c308
 
 PODFILE CHECKSUM: 49584be049764895189f1f88ebc9769116621103

package/ios/Core/CameraSession+Orientation.swift

@@ -60,6 +60,11 @@ extension CameraSession: OrientationManagerDelegate {
         connection.orientation = previewOrientation
       }
     }
+
+    // Code Scanner coordinates are relative to Preview Orientation
+    if let codeScannerOutput {
+      codeScannerOutput.orientation = previewOrientation
+    }
   }
 
   /**

package/ios/Core/CameraSession.swift

@@ -290,8 +290,7 @@ final class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegat
 
     if let delegate {
       // Call Frame Processor (delegate) for every Video Frame
-      let relativeBufferOrientation = orientation.relativeTo(orientation: outputOrientation)
-      delegate.onFrame(sampleBuffer: sampleBuffer, orientation: relativeBufferOrientation)
+      delegate.onFrame(sampleBuffer: sampleBuffer, orientation: orientation)
     }
   }
 

package/ios/React/CameraView+TakeSnapshot.swift

@@ -15,14 +15,14 @@ extension CameraView {
       guard let snapshot = latestVideoFrame else {
         throw CameraError.capture(.snapshotFailed)
       }
-      guard let imageBuffer = CMSampleBufferGetImageBuffer(snapshot) else {
+      guard let imageBuffer = CMSampleBufferGetImageBuffer(snapshot.imageBuffer) else {
         throw CameraError.capture(.imageDataAccessError)
       }
 
       self.onCaptureShutter(shutterType: .snapshot)
 
       let ciImage = CIImage(cvPixelBuffer: imageBuffer)
-      let orientation = self.cameraSession.outputOrientation
+      let orientation = Orientation.portrait.relativeTo(orientation: snapshot.orientation)
       let image = UIImage(ciImage: ciImage, scale: 1.0, orientation: orientation.imageOrientation)
       let path = try FileUtils.writeUIImageToTempFile(image: image)
       return [

package/ios/React/CameraView.swift

@@ -107,7 +107,7 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat
   var pinchScaleOffset: CGFloat = 1.0
 
   // CameraView+TakeSnapshot
-  var latestVideoFrame: CMSampleBuffer?
+  var latestVideoFrame: Snapshot?
 
   // pragma MARK: Setup
 
@@ -362,7 +362,7 @@ public final class CameraView: UIView, CameraSessionDelegate, PreviewViewDelegat
 
   func onFrame(sampleBuffer: CMSampleBuffer, orientation: Orientation) {
     // Update latest frame that can be used for snapshot capture
-    latestVideoFrame = sampleBuffer
+    latestVideoFrame = Snapshot(imageBuffer: sampleBuffer, orientation: orientation)
 
     // Notify FPS Collector that we just had a Frame
     fpsSampleCollector.onTick()

package/ios/React/Snapshot.swift

@@ -0,0 +1,14 @@
+//
+//  Snapshot.swift
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 12.07.24.
+//
+
+import AVFoundation
+import Foundation
+
+struct Snapshot {
+  let imageBuffer: CMSampleBuffer
+  let orientation: Orientation
+}

package/src/Camera.tsx

@@ -553,6 +553,11 @@ export class Camera extends React.PureComponent<CameraProps, CameraState> {
     this.rotationHelper.previewOrientation = previewOrientation
     this.props.onPreviewOrientationChanged?.(previewOrientation)
     this.maybeUpdateUIRotation()
+
+    if (isSkiaFrameProcessor(this.props.frameProcessor)) {
+      // If we have a Skia Frame Processor, we need to update it's orientation so it knows how to render.
+      this.props.frameProcessor.previewOrientation.value = previewOrientation
+    }
   }
 
   private maybeUpdateUIRotation(): void {

package/src/skia/useSkiaFrameProcessor.ts

@@ -8,6 +8,7 @@ import { WorkletsProxy } from '../dependencies/WorkletsProxy'
 import { SkiaProxy } from '../dependencies/SkiaProxy'
 import { withFrameRefCounting } from '../frame-processors/withFrameRefCounting'
 import { VisionCameraProxy } from '../frame-processors/VisionCameraProxy'
+import type { Orientation } from '../types/Orientation'
 
 /**
  * Represents a Camera Frame that can be directly drawn to using Skia.
@@ -46,19 +47,49 @@ type SurfaceCache = Record<
   }
 >
 
+function getDegrees(orientation: Orientation): number {
+  'worklet'
+  switch (orientation) {
+    case 'portrait':
+      return 0
+    case 'landscape-left':
+      return 90
+    case 'portrait-upside-down':
+      return 180
+    case 'landscape-right':
+      return 270
+  }
+}
+
+function getOrientation(degrees: number): Orientation {
+  'worklet'
+  const clamped = (degrees + 360) % 360
+  if (clamped >= 315 || clamped <= 45) return 'portrait'
+  else if (clamped >= 45 && clamped <= 135) return 'landscape-left'
+  else if (clamped >= 135 && clamped <= 225) return 'portrait-upside-down'
+  else if (clamped >= 225 && clamped <= 315) return 'landscape-right'
+  else throw new Error(`Invalid degrees! ${degrees}`)
+}
+
+function relativeTo(a: Orientation, b: Orientation): Orientation {
+  'worklet'
+  return getOrientation(getDegrees(a) - getDegrees(b))
+}
+
 /**
  * Counter-rotates the {@linkcode canvas} by the {@linkcode frame}'s {@linkcode Frame.orientation orientation}
  * to ensure the Frame will be drawn upright.
  */
-function withRotatedFrame(frame: Frame, canvas: SkCanvas, func: () => void): void {
+function withRotatedFrame(frame: Frame, canvas: SkCanvas, previewOrientation: Orientation, func: () => void): void {
   'worklet'
 
   // 1. save current translation matrix
   canvas.save()
 
   try {
     // 2. properly rotate canvas so Frame is rendered up-right.
-    switch (frame.orientation) {
+    const orientation = relativeTo(frame.orientation, previewOrientation)
+    switch (orientation) {
       case 'portrait':
         // do nothing
         break
@@ -139,6 +170,7 @@ export function createSkiaFrameProcessor(
   frameProcessor: (frame: DrawableFrame) => void,
   surfaceHolder: ISharedValue<SurfaceCache>,
   offscreenTextures: ISharedValue<SkImage[]>,
+  previewOrientation: ISharedValue<Orientation>,
 ): DrawableFrameProcessor {
   const Skia = SkiaProxy.Skia
   const Worklets = WorkletsProxy.Worklets
@@ -236,7 +268,7 @@ export function createSkiaFrameProcessor(
         canvas.clear(black)
 
         // 4. rotate the frame properly to make sure it's upright
-        withRotatedFrame(frame, canvas, () => {
+        withRotatedFrame(frame, canvas, previewOrientation.value, () => {
           // 5. Run any user drawing operations
           frameProcessor(drawableFrame)
         })
@@ -264,6 +296,7 @@ export function createSkiaFrameProcessor(
     }),
     type: 'drawable-skia',
     offscreenTextures: offscreenTextures,
+    previewOrientation: previewOrientation,
   }
 }
 
@@ -301,6 +334,7 @@ export function useSkiaFrameProcessor(
 ): DrawableFrameProcessor {
   const surface = WorkletsProxy.useSharedValue<SurfaceCache>({})
   const offscreenTextures = WorkletsProxy.useSharedValue<SkImage[]>([])
+  const previewOrientation = WorkletsProxy.useSharedValue<Orientation>('portrait')
 
   useEffect(() => {
     return () => {
@@ -322,7 +356,7 @@ export function useSkiaFrameProcessor(
   }, [offscreenTextures, surface])
 
   return useMemo(
-    () => createSkiaFrameProcessor(frameProcessor, surface, offscreenTextures),
+    () => createSkiaFrameProcessor(frameProcessor, surface, offscreenTextures, previewOrientation),
     // eslint-disable-next-line react-hooks/exhaustive-deps
     dependencies,
   )

package/src/types/CameraProps.ts

@@ -16,6 +16,7 @@ export interface DrawableFrameProcessor {
   frameProcessor: (frame: Frame) => void
   type: 'drawable-skia'
   offscreenTextures: ISharedValue<SkImage[]>
+  previewOrientation: ISharedValue<Orientation>
 }
 
 export interface OnShutterEvent {