Mocap IK Fix

packages/client/src/route/public.tsx

@@ -103,10 +103,6 @@ function RouterComp() {
       return routesReady.set(true)
   }, [clientSettingsState.client.length, authSettingsState.authSettings.length, customRoutes])
 
-  if (!(Engine.instance.api as FeathersClient).authentication?.authenticated) {
-    return <LoadingCircle message={t('common:loader.authenticating')} />
-  }
-
   if (!routesReady.value) {
     return <LoadingCircle message={t('common:loader.loadingRoutes')} />
   }

packages/engine/src/mocap/MotionCaptureSystem.ts

@@ -9,6 +9,7 @@ import { dispatchAction, getState } from '@etherealengine/hyperflux'
 
 import { AvatarRigComponent } from '../avatar/components/AvatarAnimationComponent'
 import { RingBuffer } from '../common/classes/RingBuffer'
+import { isClient } from '../common/functions/getEnvironment'
 import { Engine } from '../ecs/classes/Engine'
 import { EngineState } from '../ecs/classes/EngineState'
 import { getComponent } from '../ecs/functions/ComponentFunctions'
@@ -68,7 +69,7 @@ const handleMocapData = (
   const { peerID, landmarks } = MotionCaptureFunctions.receiveResults(message as ArrayBuffer)
   if (!peerID) return
   if (!timeSeriesMocapData.has(peerID)) {
-    timeSeriesMocapData.set(peerID, new RingBuffer(100))
+    timeSeriesMocapData.set(peerID, new RingBuffer(10))
   }
   timeSeriesMocapData.get(peerID)!.add(landmarks)
 }
@@ -90,27 +91,41 @@ const leftHandPos = new Vector3()
 const rightHandPos = new Vector3()
 
 const execute = () => {
-  const xrState = getState(XRState)
   const engineState = getState(EngineState)
 
-  if (xrState.sessionActive) return
+  const localClientEntity = Engine.instance.localClientEntity
 
   const network = Engine.instance.worldNetwork
-  if (!network) return
-
-  const localClientEntity = Engine.instance.localClientEntity
 
   for (const [peerID, mocapData] of timeSeriesMocapData) {
-    if (!network.peers.has(peerID)) {
+    if (!network?.peers?.has(peerID) || mocapData.empty()) {
       timeSeriesMocapData.delete(peerID)
-      continue
     }
+  }
+
+  const userPeers = network?.users?.get(Engine.instance.userId)
+
+  // Stop mocap by removing entities if data doesnt exist
+  if (isClient && (!localClientEntity || !userPeers?.find((peerID) => timeSeriesMocapData.has(peerID)))) {
+    const headUUID = (Engine.instance.userId + motionCaptureHeadSuffix) as EntityUUID
+    const leftHandUUID = (Engine.instance.userId + motionCaptureLeftHandSuffix) as EntityUUID
+    const rightHandUUID = (Engine.instance.userId + motionCaptureRightHandSuffix) as EntityUUID
+
+    const ikTargetHead = UUIDComponent.entitiesByUUID[headUUID]
+    const ikTargetLeftHand = UUIDComponent.entitiesByUUID[leftHandUUID]
+    const ikTargetRightHand = UUIDComponent.entitiesByUUID[rightHandUUID]
+
+    if (ikTargetHead) removeEntity(ikTargetHead)
+    if (ikTargetLeftHand) removeEntity(ikTargetLeftHand)
+    if (ikTargetRightHand) removeEntity(ikTargetRightHand)
+  }
+
+  for (const [peerID, mocapData] of timeSeriesMocapData) {
     const userID = network.peers.get(peerID)!.userId
     const entity = Engine.instance.getUserAvatarEntity(userID)
-    if (!entity) continue
 
-    if (entity === localClientEntity) {
-      const data = mocapData.getLast()
+    if (entity && entity === localClientEntity) {
+      const data = mocapData.popLast()
       if (!data) continue
 
       const leftHips = data[POSE_LANDMARKS.LEFT_HIP]
@@ -129,9 +144,9 @@ const execute = () => {
       const leftHand = !!leftWrist.visibility && leftWrist.visibility > 0.1
       const rightHand = !!rightWrist.visibility && rightWrist.visibility > 0.1
 
-      const headUUID = (Engine.instance.userId + motionCaptureHeadSuffix) as EntityUUID
-      const leftHandUUID = (Engine.instance.userId + motionCaptureLeftHandSuffix) as EntityUUID
-      const rightHandUUID = (Engine.instance.userId + motionCaptureRightHandSuffix) as EntityUUID
+      const headUUID = (userID + motionCaptureHeadSuffix) as EntityUUID
+      const leftHandUUID = (userID + motionCaptureLeftHandSuffix) as EntityUUID
+      const rightHandUUID = (userID + motionCaptureRightHandSuffix) as EntityUUID
 
       const ikTargetHead = UUIDComponent.entitiesByUUID[headUUID]
       const ikTargetLeftHand = UUIDComponent.entitiesByUUID[leftHandUUID]

packages/ui/src/pages/Capture/index.tsx

@@ -151,12 +151,12 @@ const CaptureDashboard = () => {
 
         const { poseLandmarks, faceLandmarks, leftHandLandmarks, rightHandLandmarks } = results
 
-        sendResults(poseLandmarks)
-        // sendResults(faceLandmarks)
-        // sendResults({
-        // ...(leftHandLandmarks !== undefined ? leftHandLandmarks : []),
-        // ...(rightHandLandmarks !== undefined ? rightHandLandmarks : [])
-        // })
+        /**
+         * Holistic model currently has no export for poseWorldLandmarks, instead as za (likely to change for new builds of the package)
+         * See https://github.com/google/mediapipe/issues/3155
+         */
+        //@ts-ignore
+        sendResults(results.za)
 
         processingFrame.set(false)