diff --git a/packages/client/src/route/public.tsx b/packages/client/src/route/public.tsx index 96778b0f69..0b2f5ad3bd 100644 --- a/packages/client/src/route/public.tsx +++ b/packages/client/src/route/public.tsx @@ -103,10 +103,6 @@ function RouterComp() { return routesReady.set(true) }, [clientSettingsState.client.length, authSettingsState.authSettings.length, customRoutes]) - if (!(Engine.instance.api as FeathersClient).authentication?.authenticated) { - return - } - if (!routesReady.value) { return } diff --git a/packages/engine/src/mocap/MotionCaptureSystem.ts b/packages/engine/src/mocap/MotionCaptureSystem.ts index 2d5c765bf1..38c362d22e 100644 --- a/packages/engine/src/mocap/MotionCaptureSystem.ts +++ b/packages/engine/src/mocap/MotionCaptureSystem.ts @@ -9,6 +9,7 @@ import { dispatchAction, getState } from '@etherealengine/hyperflux' import { AvatarRigComponent } from '../avatar/components/AvatarAnimationComponent' import { RingBuffer } from '../common/classes/RingBuffer' +import { isClient } from '../common/functions/getEnvironment' import { Engine } from '../ecs/classes/Engine' import { EngineState } from '../ecs/classes/EngineState' import { getComponent } from '../ecs/functions/ComponentFunctions' @@ -68,7 +69,7 @@ const handleMocapData = ( const { peerID, landmarks } = MotionCaptureFunctions.receiveResults(message as ArrayBuffer) if (!peerID) return if (!timeSeriesMocapData.has(peerID)) { - timeSeriesMocapData.set(peerID, new RingBuffer(100)) + timeSeriesMocapData.set(peerID, new RingBuffer(10)) } timeSeriesMocapData.get(peerID)!.add(landmarks) } @@ -90,27 +91,41 @@ const leftHandPos = new Vector3() const rightHandPos = new Vector3() const execute = () => { - const xrState = getState(XRState) const engineState = getState(EngineState) - if (xrState.sessionActive) return + const localClientEntity = Engine.instance.localClientEntity const network = Engine.instance.worldNetwork - if (!network) return - - const localClientEntity = Engine.instance.localClientEntity for (const [peerID, mocapData] of timeSeriesMocapData) { - if (!network.peers.has(peerID)) { + if (!network?.peers?.has(peerID) || mocapData.empty()) { timeSeriesMocapData.delete(peerID) - continue } + } + + const userPeers = network?.users?.get(Engine.instance.userId) + + // Stop mocap by removing entities if data doesnt exist + if (isClient && (!localClientEntity || !userPeers?.find((peerID) => timeSeriesMocapData.has(peerID)))) { + const headUUID = (Engine.instance.userId + motionCaptureHeadSuffix) as EntityUUID + const leftHandUUID = (Engine.instance.userId + motionCaptureLeftHandSuffix) as EntityUUID + const rightHandUUID = (Engine.instance.userId + motionCaptureRightHandSuffix) as EntityUUID + + const ikTargetHead = UUIDComponent.entitiesByUUID[headUUID] + const ikTargetLeftHand = UUIDComponent.entitiesByUUID[leftHandUUID] + const ikTargetRightHand = UUIDComponent.entitiesByUUID[rightHandUUID] + + if (ikTargetHead) removeEntity(ikTargetHead) + if (ikTargetLeftHand) removeEntity(ikTargetLeftHand) + if (ikTargetRightHand) removeEntity(ikTargetRightHand) + } + + for (const [peerID, mocapData] of timeSeriesMocapData) { const userID = network.peers.get(peerID)!.userId const entity = Engine.instance.getUserAvatarEntity(userID) - if (!entity) continue - if (entity === localClientEntity) { - const data = mocapData.getLast() + if (entity && entity === localClientEntity) { + const data = mocapData.popLast() if (!data) continue const leftHips = data[POSE_LANDMARKS.LEFT_HIP] @@ -129,9 +144,9 @@ const execute = () => { const leftHand = !!leftWrist.visibility && leftWrist.visibility > 0.1 const rightHand = !!rightWrist.visibility && rightWrist.visibility > 0.1 - const headUUID = (Engine.instance.userId + motionCaptureHeadSuffix) as EntityUUID - const leftHandUUID = (Engine.instance.userId + motionCaptureLeftHandSuffix) as EntityUUID - const rightHandUUID = (Engine.instance.userId + motionCaptureRightHandSuffix) as EntityUUID + const headUUID = (userID + motionCaptureHeadSuffix) as EntityUUID + const leftHandUUID = (userID + motionCaptureLeftHandSuffix) as EntityUUID + const rightHandUUID = (userID + motionCaptureRightHandSuffix) as EntityUUID const ikTargetHead = UUIDComponent.entitiesByUUID[headUUID] const ikTargetLeftHand = UUIDComponent.entitiesByUUID[leftHandUUID] diff --git a/packages/ui/src/pages/Capture/index.tsx b/packages/ui/src/pages/Capture/index.tsx index 4cd8edae85..045834c14a 100755 --- a/packages/ui/src/pages/Capture/index.tsx +++ b/packages/ui/src/pages/Capture/index.tsx @@ -151,12 +151,12 @@ const CaptureDashboard = () => { const { poseLandmarks, faceLandmarks, leftHandLandmarks, rightHandLandmarks } = results - sendResults(poseLandmarks) - // sendResults(faceLandmarks) - // sendResults({ - // ...(leftHandLandmarks !== undefined ? leftHandLandmarks : []), - // ...(rightHandLandmarks !== undefined ? rightHandLandmarks : []) - // }) + /** + * Holistic model currently has no export for poseWorldLandmarks, instead as za (likely to change for new builds of the package) + * See https://github.com/google/mediapipe/issues/3155 + */ + //@ts-ignore + sendResults(results.za) processingFrame.set(false)