diff --git a/packages/client/src/route/public.tsx b/packages/client/src/route/public.tsx
index 96778b0f69..0b2f5ad3bd 100644
--- a/packages/client/src/route/public.tsx
+++ b/packages/client/src/route/public.tsx
@@ -103,10 +103,6 @@ function RouterComp() {
return routesReady.set(true)
}, [clientSettingsState.client.length, authSettingsState.authSettings.length, customRoutes])
- if (!(Engine.instance.api as FeathersClient).authentication?.authenticated) {
- return
- }
-
if (!routesReady.value) {
return
}
diff --git a/packages/engine/src/mocap/MotionCaptureSystem.ts b/packages/engine/src/mocap/MotionCaptureSystem.ts
index 2d5c765bf1..38c362d22e 100644
--- a/packages/engine/src/mocap/MotionCaptureSystem.ts
+++ b/packages/engine/src/mocap/MotionCaptureSystem.ts
@@ -9,6 +9,7 @@ import { dispatchAction, getState } from '@etherealengine/hyperflux'
import { AvatarRigComponent } from '../avatar/components/AvatarAnimationComponent'
import { RingBuffer } from '../common/classes/RingBuffer'
+import { isClient } from '../common/functions/getEnvironment'
import { Engine } from '../ecs/classes/Engine'
import { EngineState } from '../ecs/classes/EngineState'
import { getComponent } from '../ecs/functions/ComponentFunctions'
@@ -68,7 +69,7 @@ const handleMocapData = (
const { peerID, landmarks } = MotionCaptureFunctions.receiveResults(message as ArrayBuffer)
if (!peerID) return
if (!timeSeriesMocapData.has(peerID)) {
- timeSeriesMocapData.set(peerID, new RingBuffer(100))
+ timeSeriesMocapData.set(peerID, new RingBuffer(10))
}
timeSeriesMocapData.get(peerID)!.add(landmarks)
}
@@ -90,27 +91,41 @@ const leftHandPos = new Vector3()
const rightHandPos = new Vector3()
const execute = () => {
- const xrState = getState(XRState)
const engineState = getState(EngineState)
- if (xrState.sessionActive) return
+ const localClientEntity = Engine.instance.localClientEntity
const network = Engine.instance.worldNetwork
- if (!network) return
-
- const localClientEntity = Engine.instance.localClientEntity
for (const [peerID, mocapData] of timeSeriesMocapData) {
- if (!network.peers.has(peerID)) {
+ if (!network?.peers?.has(peerID) || mocapData.empty()) {
timeSeriesMocapData.delete(peerID)
- continue
}
+ }
+
+ const userPeers = network?.users?.get(Engine.instance.userId)
+
+ // Stop mocap by removing entities if data doesnt exist
+ if (isClient && (!localClientEntity || !userPeers?.find((peerID) => timeSeriesMocapData.has(peerID)))) {
+ const headUUID = (Engine.instance.userId + motionCaptureHeadSuffix) as EntityUUID
+ const leftHandUUID = (Engine.instance.userId + motionCaptureLeftHandSuffix) as EntityUUID
+ const rightHandUUID = (Engine.instance.userId + motionCaptureRightHandSuffix) as EntityUUID
+
+ const ikTargetHead = UUIDComponent.entitiesByUUID[headUUID]
+ const ikTargetLeftHand = UUIDComponent.entitiesByUUID[leftHandUUID]
+ const ikTargetRightHand = UUIDComponent.entitiesByUUID[rightHandUUID]
+
+ if (ikTargetHead) removeEntity(ikTargetHead)
+ if (ikTargetLeftHand) removeEntity(ikTargetLeftHand)
+ if (ikTargetRightHand) removeEntity(ikTargetRightHand)
+ }
+
+ for (const [peerID, mocapData] of timeSeriesMocapData) {
const userID = network.peers.get(peerID)!.userId
const entity = Engine.instance.getUserAvatarEntity(userID)
- if (!entity) continue
- if (entity === localClientEntity) {
- const data = mocapData.getLast()
+ if (entity && entity === localClientEntity) {
+ const data = mocapData.popLast()
if (!data) continue
const leftHips = data[POSE_LANDMARKS.LEFT_HIP]
@@ -129,9 +144,9 @@ const execute = () => {
const leftHand = !!leftWrist.visibility && leftWrist.visibility > 0.1
const rightHand = !!rightWrist.visibility && rightWrist.visibility > 0.1
- const headUUID = (Engine.instance.userId + motionCaptureHeadSuffix) as EntityUUID
- const leftHandUUID = (Engine.instance.userId + motionCaptureLeftHandSuffix) as EntityUUID
- const rightHandUUID = (Engine.instance.userId + motionCaptureRightHandSuffix) as EntityUUID
+ const headUUID = (userID + motionCaptureHeadSuffix) as EntityUUID
+ const leftHandUUID = (userID + motionCaptureLeftHandSuffix) as EntityUUID
+ const rightHandUUID = (userID + motionCaptureRightHandSuffix) as EntityUUID
const ikTargetHead = UUIDComponent.entitiesByUUID[headUUID]
const ikTargetLeftHand = UUIDComponent.entitiesByUUID[leftHandUUID]
diff --git a/packages/ui/src/pages/Capture/index.tsx b/packages/ui/src/pages/Capture/index.tsx
index 4cd8edae85..045834c14a 100755
--- a/packages/ui/src/pages/Capture/index.tsx
+++ b/packages/ui/src/pages/Capture/index.tsx
@@ -151,12 +151,12 @@ const CaptureDashboard = () => {
const { poseLandmarks, faceLandmarks, leftHandLandmarks, rightHandLandmarks } = results
- sendResults(poseLandmarks)
- // sendResults(faceLandmarks)
- // sendResults({
- // ...(leftHandLandmarks !== undefined ? leftHandLandmarks : []),
- // ...(rightHandLandmarks !== undefined ? rightHandLandmarks : [])
- // })
+ /**
+ * Holistic model currently has no export for poseWorldLandmarks, instead as za (likely to change for new builds of the package)
+ * See https://github.com/google/mediapipe/issues/3155
+ */
+ //@ts-ignore
+ sendResults(results.za)
processingFrame.set(false)