Migrate to VideoKit

natmlx · Nov 16, 2022 · b64f054 · b64f054
1 parent 93f5fd6
commit b64f054
Show file tree

Hide file tree

Showing 8 changed files with 459 additions and 426 deletions.
diff --git a/Assets/BlazePoseSample.cs b/Assets/BlazePoseSample.cs
@@ -8,54 +8,47 @@ namespace NatML.Examples {
     using UnityEngine;
     using NatML.Devices;
     using NatML.Devices.Outputs;
+    using NatML.VideoKit;
     using NatML.Vision;
     using Visualizers;
 
     public sealed class BlazePoseSample : MonoBehaviour {
 
+        [Header(@"VideoKit")]
+        public VideoKitCameraManager cameraManager;
+
         [Header(@"Visualizers")]
         public LandmarkVisualizer visualizer;
         public Landmark3DVisualizer visualizer3D;
 
-        CameraDevice cameraDevice;
-        TextureOutput cameraTextureOutput;
-        BlazePosePipeline posePipeline;
+        private BlazePosePipeline pipeline;
 
-        async void Start () {
-            // Request camera permissions
-            var permissionStatus = await MediaDeviceQuery.RequestPermissions<CameraDevice>();
-            if (permissionStatus != PermissionStatus.Authorized) {
-                Debug.Log(@"User did not grant camera permissions");
-                return;
-            }
-            // Get a camera device
-            var query = new MediaDeviceQuery(MediaDeviceCriteria.CameraDevice);
-            cameraDevice = query.current as CameraDevice;
-            // Start the camera preview
-            cameraTextureOutput = new TextureOutput();
-            cameraDevice.StartRunning(cameraTextureOutput);
-            // Display the preview
-            await cameraTextureOutput.textureCreated;
-            visualizer.image = cameraTextureOutput.texture;
+        private async void Start () {
             // Create the BlazePose pipeline
             var detectorModelData = await MLModelData.FromHub("@natml/blazepose-detector");
             var predictorModelData = await MLModelData.FromHub("@natml/blazepose-landmark");
-            posePipeline = new BlazePosePipeline(detectorModelData, predictorModelData, maxDetections: 1);
+            pipeline = new BlazePosePipeline(detectorModelData, predictorModelData, maxDetections: 1);
+            // Listen for camera frames
+            cameraManager.OnFrame.AddListener(OnCameraFrame);
         }
 
-        void Update () {
-            // Check that pipeline has been created
-            if (posePipeline == null)
-                return;
+        private void OnCameraFrame (CameraFrame frame) {
             // Predict
-            var poses = posePipeline.Predict(cameraTextureOutput.texture);
-            // Visualize
+            var poses = pipeline.Predict(frame);
+            // Check
             if (poses.Length == 0)
                 return;
+            // Visualize
             var pose = poses[0];
-            visualizer.Clear();
-            visualizer.Render(pose.keypoints, Color.green);
+            visualizer.Render(pose.keypoints);
             visualizer3D.Render(pose.keypoints3D);
         }
+
+        private void OnDisable () {
+            // Stop listening for camera frames
+            cameraManager.OnFrame.RemoveListener(OnCameraFrame);
+            // Dispose the pipeline
+            pipeline?.Dispose();
+        }
     }
 }
diff --git a/Assets/BlazePoseSample.unity b/Assets/BlazePoseSample.unity
@@ -202,6 +202,7 @@ GameObject:
   - component: {fileID: 357584624}
   - component: {fileID: 357584623}
   - component: {fileID: 357584625}
+  - component: {fileID: 357584627}
   - component: {fileID: 357584626}
   m_Layer: 5
   m_Name: Preview
@@ -292,6 +293,20 @@ MonoBehaviour:
   m_Name: 
   m_EditorClassIdentifier: 
   keypoint: {fileID: 878973901}
+--- !u!114 &357584627
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 357584621}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 145a5a333d6e14379ae91956c0b3373e, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  cameraManager: {fileID: 2111620525}
+  viewMode: 0
 --- !u!1 &705507993
 GameObject:
   m_ObjectHideFlags: 0
@@ -1025,6 +1040,7 @@ GameObject:
   serializedVersion: 6
   m_Component:
   - component: {fileID: 2111620523}
+  - component: {fileID: 2111620525}
   - component: {fileID: 2111620524}
   m_Layer: 0
   m_Name: BlazePoseSample
@@ -1060,5 +1076,26 @@ MonoBehaviour:
   m_Script: {fileID: 11500000, guid: 0dc1abe2732cf4c29bd30fa0833535e3, type: 3}
   m_Name: 
   m_EditorClassIdentifier: 
+  cameraManager: {fileID: 2111620525}
   visualizer: {fileID: 357584626}
   visualizer3D: {fileID: 1522798682}
+--- !u!114 &2111620525
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 2111620522}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: 98622613d505143a9a06b4b11b0a82cf, type: 3}
+  m_Name: 
+  m_EditorClassIdentifier: 
+  resolution: 3
+  capabilities: 1
+  playOnAwake: 1
+  focusMode: 0
+  exposureMode: 0
+  OnFrame:
+    m_PersistentCalls:
+      m_Calls: []
diff --git a/Assets/LandmarkVisualizer.cs b/Assets/LandmarkVisualizer.cs
@@ -8,11 +8,12 @@ namespace NatML.Examples.Visualizers {
     using System.Collections.Generic;
     using UnityEngine;
     using UnityEngine.UI;
+    using NatML.VideoKit.UI;
 
     /// <summary>
     /// Lightweight 2D pose visualizer.
     /// </summary>
-    [RequireComponent(typeof(RawImage), typeof(AspectRatioFitter))]
+    [RequireComponent(typeof(VideoKitCameraView))]
     public sealed class LandmarkVisualizer : MonoBehaviour {
 
         #region --Inspector--
@@ -24,56 +25,30 @@ public sealed class LandmarkVisualizer : MonoBehaviour {
 
 
         #region --Client API--
-        /// <summary>
-        /// Get or set the detection image.
-        /// </summary>
-        public Texture2D image {
-            get => rawImage.texture as Texture2D;
-            set {
-                rawImage.texture = value;
-                aspectFitter.aspectRatio = (float)value.width / value.height;
-            }
-        }
-
         /// <summary>
         /// Render a 2D pose.
         /// </summary>
         /// <param name="pose">Pose keypoints in normalized coordinates.</param>
         /// <param name="color">Pose keypoint color.</param>
-        public void Render (IEnumerable<Vector4> pose, Color color) {
-            foreach (var point in pose)
-                AddKeypoint(point, color);
-        }
-
-        /// <summary>
-        /// Clear all rendered poses.
-        /// </summary>
-        public void Clear () {
-            foreach (var obj in current)
-                GameObject.Destroy(obj);
-            current.Clear();
+        public void Render (params IEnumerable<Vector4>[] poses) {
+            foreach (var pose in poses)
+                foreach (var point in pose)
+                    AddKeypoint(point, Color.green);
         }
         #endregion
 
 
         #region --Operations--
-        private RawImage rawImage;
-        private AspectRatioFitter aspectFitter;
         private readonly List<GameObject> current = new List<GameObject>();
 
-        private void Awake () {
-            rawImage = GetComponent<RawImage>();
-            aspectFitter = GetComponent<AspectRatioFitter>();
-        }
-
         private void AddKeypoint (Vector2 point, Color color) {
             // Instantiate
             var prefab = Instantiate(keypoint, transform);
             prefab.gameObject.SetActive(true);
             prefab.color = color;
             // Position
             var prefabTransform = prefab.transform as RectTransform;
-            var imageTransform = rawImage.transform as RectTransform;
+            var imageTransform = transform as RectTransform;
             prefabTransform.anchorMin = 0.5f * Vector2.one;
             prefabTransform.anchorMax = 0.5f * Vector2.one;
             prefabTransform.pivot = 0.5f * Vector2.one;

diff --git a/Packages/manifest.json b/Packages/manifest.json
@@ -12,7 +12,7 @@
     }
   ],
   "dependencies": {
-    "ai.natml.natdevice": "1.3.1",
+    "ai.natml.videokit": "0.0.3",
     "com.unity.collab-proxy": "1.15.17",
     "com.unity.feature.development": "1.0.1",
     "com.unity.ide.rider": "3.0.14",

diff --git a/Packages/packages-lock.json b/Packages/packages-lock.json
@@ -2,14 +2,23 @@
   "dependencies": {
     "ai.natml.hub": {
       "version": "1.0.15",
-      "depth": 1,
+      "depth": 2,
       "source": "registry",
       "dependencies": {},
       "url": "https://registry.npmjs.com"
     },
+    "ai.natml.natcorder": {
+      "version": "1.9.1",
+      "depth": 1,
+      "source": "registry",
+      "dependencies": {
+        "ai.natml.hub": "1.0.15"
+      },
+      "url": "https://registry.npmjs.com"
+    },
     "ai.natml.natdevice": {
       "version": "1.3.1",
-      "depth": 0,
+      "depth": 1,
       "source": "registry",
       "dependencies": {
         "ai.natml.hub": "1.0.15"
@@ -25,6 +34,27 @@
       },
       "url": "https://registry.npmjs.com"
     },
+    "ai.natml.natshare": {
+      "version": "1.3.0",
+      "depth": 1,
+      "source": "registry",
+      "dependencies": {
+        "ai.natml.hub": "1.0.12"
+      },
+      "url": "https://registry.npmjs.com"
+    },
+    "ai.natml.videokit": {
+      "version": "0.0.3",
+      "depth": 0,
+      "source": "registry",
+      "dependencies": {
+        "ai.natml.natcorder": "1.9.1",
+        "ai.natml.natdevice": "1.3.1",
+        "ai.natml.natml": "1.0.19",
+        "ai.natml.natshare": "1.3.0"
+      },
+      "url": "https://registry.npmjs.com"
+    },
     "ai.natml.vision.blazepose": {
       "version": "file:ai.natml.vision.blazepose",
       "depth": 0,

diff --git a/README.md b/README.md
@@ -17,7 +17,7 @@ Retrieve your access key from [NatML Hub](https://hub.natml.ai/profile) and add
 
 ## Supported Platforms
 - Android API level 24+
-- iOS 13+
+- iOS 14+
 - macOS 10.15+ (Apple Silicon and Intel)
 - Windows 10+ (64-bit only)
 - WebGL:
@@ -27,11 +27,9 @@ Retrieve your access key from [NatML Hub](https://hub.natml.ai/profile) and add
 ## Resources
 - Join the [NatML community on Discord](https://hub.natml.ai/community).
 - See the [NatML documentation](https://docs.natml.ai/unity).
-- See the [NatDevice documentation](https://docs.natml.ai/natdevice).
+- See the [VideoKit documentation](https://docs.natml.ai/videokit).
 - Check out [NatML on GitHub](https://github.com/natmlx).
 - Read the [NatML blog](https://blog.natml.ai/).
-- Discuss [NatML on Unity Forums](https://forum.unity.com/threads/open-beta-natml-machine-learning-runtime.1109339/).
-- Discuss [NatDevice on Unity Forums](https://forum.unity.com/threads/natdevice-media-device-api.374690/).
 - Contact us at [hi@natml.ai](mailto:hi@natml.ai).
 
 Thank you very much!