Rig Size Component for Root Motion Scale Correction

packages/client-core/src/media/webcam/WebcamInput.ts

@@ -43,6 +43,7 @@ import { GroupComponent } from '@etherealengine/engine/src/scene/components/Grou
 import { UUIDComponent } from '@etherealengine/engine/src/scene/components/UUIDComponent'
 import { defineActionQueue, getMutableState } from '@etherealengine/hyperflux'
 
+import { AvatarComponent } from '@etherealengine/engine/src/avatar/components/AvatarComponent'
 import { AvatarNetworkAction } from '@etherealengine/engine/src/avatar/state/AvatarNetworkActions'
 import { AnimationSystem } from '@etherealengine/engine/src/avatar/systems/AnimationSystem'
 import { MediaStreamState } from '../../transports/MediaStreams'
@@ -259,7 +260,7 @@ const setAvatarExpression = (entity: Entity): void => {
   if (morphValue === 0) return
 
   const morphName = morphNameByIndex[WebcamInputComponent.expressionIndex[entity]]
-  const skinnedMeshes = getComponent(entity, AvatarRigComponent).skinnedMeshes
+  const skinnedMeshes = getComponent(entity, AvatarComponent).skinnedMeshes
 
   for (const obj of skinnedMeshes) {
     if (!obj.morphTargetDictionary || !obj.morphTargetInfluences) continue

packages/engine/src/avatar/AvatarBoneMatching.ts

@@ -673,7 +673,6 @@ export default function avatarBoneMatching(asset: VRM | GLTF): VRM | GLTF {
    * that must be removed for matching to keys in the mixamoVRMRigMap
    */
   const removeSuffix = mixamoPrefix ? false : !/[hp]/i.test(hips.name.charAt(9))
-
   hips.traverse((target) => {
     /**match the keys to create a humanoid bones object */
     let boneName = mixamoPrefix + target.name

packages/engine/src/avatar/components/AvatarAnimationComponent.ts

@@ -25,7 +25,7 @@ Ethereal Engine. All Rights Reserved.
 
 import { VRM, VRMHumanBones } from '@pixiv/three-vrm'
 import { useEffect } from 'react'
-import { AnimationAction, Euler, KeyframeTrack, Matrix4, Quaternion, SkeletonHelper, SkinnedMesh, Vector3 } from 'three'
+import { AnimationAction, Euler, KeyframeTrack, Matrix4, Quaternion, SkeletonHelper, Vector3 } from 'three'
 
 import { getMutableState, none, useHookstate } from '@etherealengine/hyperflux'
 
@@ -101,24 +101,9 @@ export const AvatarRigComponent = defineComponent({
       rawRig: null! as VRMHumanBones,
 
       helperEntity: null as Entity | null,
-      /** The length of the torso in a t-pose, from the hip joint to the head joint */
-      torsoLength: 0,
-      /** The length of the upper leg in a t-pose, from the hip joint to the knee joint */
-      upperLegLength: 0,
-      /** The length of the lower leg in a t-pose, from the knee joint to the ankle joint */
-      lowerLegLength: 0,
-      /** The height of the foot in a t-pose, from the ankle joint to the bottom of the avatar's model */
-      footHeight: 0,
 
-      armLength: 0,
-
-      footGap: 0,
-
-      /** Cache of the skinned meshes currently on the rig */
-      skinnedMeshes: [] as SkinnedMesh[],
       /** The VRM model */
       vrm: null! as VRM,
-
       avatarURL: null as string | null
     }
   },
@@ -127,12 +112,6 @@ export const AvatarRigComponent = defineComponent({
     if (!json) return
     if (matches.object.test(json.normalizedRig)) component.normalizedRig.set(json.normalizedRig)
     if (matches.object.test(json.rawRig)) component.rawRig.set(json.rawRig)
-    if (matches.number.test(json.torsoLength)) component.torsoLength.set(json.torsoLength)
-    if (matches.number.test(json.upperLegLength)) component.upperLegLength.set(json.upperLegLength)
-    if (matches.number.test(json.lowerLegLength)) component.lowerLegLength.set(json.lowerLegLength)
-    if (matches.number.test(json.footHeight)) component.footHeight.set(json.footHeight)
-    if (matches.number.test(json.footGap)) component.footGap.set(json.footGap)
-    if (matches.array.test(json.skinnedMeshes)) component.skinnedMeshes.set(json.skinnedMeshes as SkinnedMesh[])
     if (matches.object.test(json.vrm)) component.vrm.set(json.vrm as VRM)
     if (matches.string.test(json.avatarURL)) component.avatarURL.set(json.avatarURL)
   },
@@ -150,6 +129,7 @@ export const AvatarRigComponent = defineComponent({
     const visible = useOptionalComponent(entity, VisibleComponent)
     const modelComponent = useOptionalComponent(entity, ModelComponent)
     const locomotionAnimationState = useHookstate(getMutableState(AnimationState).loadedAnimations[locomotionAnimation])
+    const avatarComponent = useComponent(entity, AvatarComponent)
 
     useEffect(() => {
       if (!visible?.value || !debugEnabled.value || pending?.value || !rigComponent.value.normalizedRig?.hips?.node)

packages/engine/src/avatar/components/AvatarComponent.ts

@@ -23,22 +23,84 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20
 Ethereal Engine. All Rights Reserved.
 */
 
+import { useEffect } from 'react'
+import { Box3, SkinnedMesh, Vector3 } from 'three'
 import { matches } from '../../common/functions/MatchesUtils'
-import { defineComponent } from '../../ecs/functions/ComponentFunctions'
+import { defineComponent, getOptionalComponent, useComponent } from '../../ecs/functions/ComponentFunctions'
+import { useEntityContext } from '../../ecs/functions/EntityFunctions'
+import { EntityTreeComponent } from '../../ecs/functions/EntityTree'
+import { ModelComponent } from '../../scene/components/ModelComponent'
+import { SkinnedMeshComponent } from './SkinnedMeshComponent'
 
+const size = new Vector3()
 export const AvatarComponent = defineComponent({
   name: 'AvatarComponent',
 
   onInit: (entity) => {
     return {
       avatarHeight: 0,
-      avatarHalfHeight: 0
+      avatarHalfHeight: 0,
+      /** The length of the torso in a t-pose, from the hip joint to the head joint */
+      torsoLength: 0,
+      /** The length of the upper leg in a t-pose, from the hip joint to the knee joint */
+      upperLegLength: 0,
+      /** The length of the lower leg in a t-pose, from the knee joint to the ankle joint */
+      lowerLegLength: 0,
+      /** The height of the foot in a t-pose, from the ankle joint to the bottom of the avatar's model */
+      footHeight: 0,
+      /** The height of the hips in a t-pose */
+      hipsHeight: 0,
+      /** The length of the arm in a t-pose, from the shoulder joint to the elbow joint */
+      armLength: 0,
+      /** The distance between the left and right foot in a t-pose */
+      footGap: 0,
+      /** The height of the eyes in a t-pose */
+      eyeHeight: 0,
+
+      skinnedMeshes: [] as SkinnedMesh[]
     }
   },
 
   onSet: (entity, component, json) => {
     if (!json) return
     if (matches.number.test(json.avatarHeight)) component.avatarHeight.set(json.avatarHeight)
     if (matches.number.test(json.avatarHalfHeight)) component.avatarHalfHeight.set(json.avatarHalfHeight)
+    if (matches.number.test(json.torsoLength)) component.torsoLength.set(json.torsoLength)
+    if (matches.number.test(json.upperLegLength)) component.upperLegLength.set(json.upperLegLength)
+    if (matches.number.test(json.lowerLegLength)) component.lowerLegLength.set(json.lowerLegLength)
+    if (matches.number.test(json.footHeight)) component.footHeight.set(json.footHeight)
+    if (matches.number.test(json.hipsHeight)) component.hipsHeight.set(json.hipsHeight)
+    if (matches.number.test(json.footGap)) component.footGap.set(json.footGap)
+    if (matches.number.test(json.eyeHeight)) component.eyeHeight.set(json.eyeHeight)
+  },
+
+  reactor: () => {
+    const entity = useEntityContext()
+    const avatarComponent = useComponent(entity, AvatarComponent)
+    const modelComponent = useComponent(entity, ModelComponent)
+    const entityTreeComponent = useComponent(entity, EntityTreeComponent)
+
+    useEffect(() => {
+      if (!modelComponent.asset.value) return
+      const scene = modelComponent.asset.value.scene
+      if (!scene) return
+      const box = new Box3()
+      box.expandByObject(scene).getSize(size)
+      avatarComponent.avatarHeight.set(size.y)
+      avatarComponent.avatarHalfHeight.set(size.y * 0.5)
+    }, [modelComponent.asset])
+
+    useEffect(() => {
+      const children = entityTreeComponent.children.value
+      if (!children.length) return
+      const skinnedMeshes = [] as SkinnedMesh[]
+      for (const child of children) {
+        const skinnedMesh = getOptionalComponent(child, SkinnedMeshComponent)
+        if (skinnedMesh) skinnedMeshes.push(skinnedMesh)
+      }
+      avatarComponent.skinnedMeshes.set(skinnedMeshes)
+    }, [entityTreeComponent.children])
+
+    return null
   }
 })

packages/engine/src/avatar/components/AvatarControllerComponent.ts

@@ -26,10 +26,17 @@ Ethereal Engine. All Rights Reserved.
 import { Collider, KinematicCharacterController } from '@dimforge/rapier3d-compat'
 import { Vector3 } from 'three'
 
+import { getState } from '@etherealengine/hyperflux'
+import { useEffect } from 'react'
 import { matches } from '../../common/functions/MatchesUtils'
 import { Engine } from '../../ecs/classes/Engine'
 import { Entity } from '../../ecs/classes/Entity'
-import { defineComponent, getComponent } from '../../ecs/functions/ComponentFunctions'
+import { defineComponent, getComponent, setComponent, useComponent } from '../../ecs/functions/ComponentFunctions'
+import { useEntityContext } from '../../ecs/functions/EntityFunctions'
+import { Physics } from '../../physics/classes/Physics'
+import { PhysicsState } from '../../physics/state/PhysicsState'
+import { createAvatarCollider } from '../functions/spawnAvatarReceptor'
+import { AvatarComponent } from './AvatarComponent'
 
 export const AvatarControllerComponent = defineComponent({
   name: 'AvatarControllerComponent',
@@ -51,11 +58,7 @@ export const AvatarControllerComponent = defineComponent({
       /** gamepad-driven input, in the local XZ plane */
       gamepadLocalInput: new Vector3(),
       /** gamepad-driven movement, in the world XZ plane */
-      gamepadWorldMovement: new Vector3(),
-      // Below two values used to smoothly transition between
-      // walk and run speeds
-      /** @todo refactor animation system */
-      speedVelocity: 0
+      gamepadWorldMovement: new Vector3()
     }
   },
 
@@ -73,7 +76,6 @@ export const AvatarControllerComponent = defineComponent({
     if (matches.boolean.test(json.gamepadJumpActive)) component.gamepadJumpActive.set(json.gamepadJumpActive)
     if (matches.object.test(json.gamepadLocalInput)) component.gamepadLocalInput.set(json.gamepadLocalInput)
     if (matches.object.test(json.gamepadWorldMovement)) component.gamepadWorldMovement.set(json.gamepadWorldMovement)
-    if (matches.number.test(json.speedVelocity)) component.speedVelocity.set(json.speedVelocity)
   },
 
   captureMovement(capturedEntity: Entity, entity: Entity): void {
@@ -86,5 +88,18 @@ export const AvatarControllerComponent = defineComponent({
     const component = getComponent(capturedEntity, AvatarControllerComponent)
     const index = component.movementCaptured.indexOf(entity)
     if (index !== -1) component.movementCaptured.splice(index, 1)
+  },
+
+  reactor: () => {
+    const entity = useEntityContext()
+    const avatarComponent = useComponent(entity, AvatarComponent)
+
+    useEffect(() => {
+      Physics.removeCollidersFromRigidBody(entity, getState(PhysicsState).physicsWorld)
+      const collider = createAvatarCollider(entity)
+      setComponent(entity, AvatarControllerComponent, { bodyCollider: collider })
+    }, [avatarComponent.avatarHeight])
+
+    return null
   }
 })

packages/engine/src/avatar/functions/avatarFootHeuristics.ts

@@ -31,7 +31,7 @@ import { getComponent } from '../../ecs/functions/ComponentFunctions'
 import { UUIDComponent } from '../../scene/components/UUIDComponent'
 import { TransformComponent } from '../../transform/components/TransformComponent'
 import { ikTargets } from '../animation/Util'
-import { AvatarRigComponent } from '../components/AvatarAnimationComponent'
+import { AvatarComponent } from '../components/AvatarComponent'
 import { AvatarIKTargetComponent } from '../components/AvatarIKComponents'
 
 const walkDirection = new Vector3()
@@ -63,8 +63,8 @@ export const setIkFootTarget = (localClientEntity: Entity, delta: number) => {
 
   if (!leftFootTargetBlendWeight || !rightFootTargetBlendWeight) return
 
-  const rigComponent = getComponent(localClientEntity, AvatarRigComponent)
-  const stepThreshold = rigComponent.upperLegLength + rigComponent.lowerLegLength
+  const avatar = getComponent(localClientEntity, AvatarComponent)
+  const stepThreshold = avatar.upperLegLength + avatar.lowerLegLength
 
   const feet = {
     [ikTargets.rightFoot]: UUIDComponent.getEntityByUUID((userID + ikTargets.rightFoot) as EntityUUID),
@@ -75,11 +75,9 @@ export const setIkFootTarget = (localClientEntity: Entity, delta: number) => {
   if (lastPlayerPosition.x == 0 && lastPlayerPosition.y == 0 && lastPlayerPosition.z == 0)
     lastPlayerPosition.copy(playerTransform.position)
 
-  const playerRig = getComponent(localClientEntity, AvatarRigComponent)
-
   /**calculate foot offset so both feet aren't at the transform's center */
   const calculateFootOffset = () => {
-    footOffset.set(currentStep == ikTargets.leftFoot ? playerRig.footGap : -playerRig.footGap, 0, 0)
+    footOffset.set(currentStep == ikTargets.leftFoot ? avatar.footGap : -avatar.footGap, 0, 0)
     footOffset.applyQuaternion(playerTransform.rotation)
     footOffset.add(playerTransform.position)
     return footOffset

packages/engine/src/avatar/functions/avatarFunctions.ts

@@ -24,14 +24,15 @@ Ethereal Engine. All Rights Reserved.
 */
 
 import { VRM, VRM1Meta, VRMHumanBone, VRMHumanoid } from '@pixiv/three-vrm'
-import { AnimationClip, AnimationMixer, Box3, Object3D, Vector3 } from 'three'
+import { AnimationClip, AnimationMixer, Vector3 } from 'three'
 
 import { getMutableState, getState } from '@etherealengine/hyperflux'
 
 import { AssetLoader } from '../../assets/classes/AssetLoader'
 import { Entity } from '../../ecs/classes/Entity'
 import {
   getComponent,
+  getMutableComponent,
   getOptionalComponent,
   hasComponent,
   removeComponent,
@@ -50,22 +51,18 @@ import { Engine } from '../../ecs/classes/Engine'
 import { EngineState } from '../../ecs/classes/EngineState'
 import { ModelComponent } from '../../scene/components/ModelComponent'
 import { XRState } from '../../xr/XRState'
-import avatarBoneMatching, { findSkinnedMeshes } from '../AvatarBoneMatching'
+import avatarBoneMatching from '../AvatarBoneMatching'
 import { getRootSpeed } from '../animation/AvatarAnimationGraph'
 import { locomotionAnimation } from '../animation/Util'
 import { AnimationComponent } from '../components/AnimationComponent'
-import { AvatarAnimationComponent, AvatarRigComponent } from '../components/AvatarAnimationComponent'
+import { AvatarRigComponent } from '../components/AvatarAnimationComponent'
 import { AvatarComponent } from '../components/AvatarComponent'
 import { AvatarControllerComponent } from '../components/AvatarControllerComponent'
 import { AvatarDissolveComponent } from '../components/AvatarDissolveComponent'
 import { AvatarPendingComponent } from '../components/AvatarPendingComponent'
 import { AvatarMovementSettingsState } from '../state/AvatarMovementSettingsState'
-import { resizeAvatar } from './resizeAvatar'
 import { bindAnimationClipFromMixamo, retargetAnimationClip } from './retargetMixamoRig'
 
-const tempVec3ForHeight = new Vector3()
-const tempVec3ForCenter = new Vector3()
-
 declare module '@pixiv/three-vrm/types/VRM' {
   export interface VRM {
     userData: {
@@ -133,12 +130,40 @@ export const unloadAvatarForUser = async (entity: Entity) => {
   removeComponent(entity, AvatarPendingComponent)
 }
 
+const hipsPos = new Vector3(),
+  headPos = new Vector3(),
+  leftFootPos = new Vector3(),
+  rightFootPos = new Vector3(),
+  leftLowerLegPos = new Vector3(),
+  leftUpperLegPos = new Vector3(),
+  footGap = new Vector3(),
+  eyePos = new Vector3()
+
 /**Kicks off avatar animation loading and setup. Called after an avatar's model asset is
  * successfully loaded.
  */
 export const setupAvatarForUser = (entity: Entity, model: VRM) => {
-  rigAvatarModel(entity)(model)
-  setupAvatarHeight(entity, model.scene)
+  setComponent(entity, AvatarRigComponent, {
+    normalizedRig: model.humanoid.normalizedHumanBones,
+    rawRig: model.humanoid.rawHumanBones
+  })
+
+  const rig = getComponent(entity, AvatarRigComponent).normalizedRig
+  rig.hips.node.getWorldPosition(hipsPos)
+  rig.head.node.getWorldPosition(headPos)
+  rig.leftFoot.node.getWorldPosition(leftFootPos)
+  rig.rightFoot.node.getWorldPosition(rightFootPos)
+  rig.leftLowerLeg.node.getWorldPosition(leftLowerLegPos)
+  rig.leftUpperLeg.node.getWorldPosition(leftUpperLegPos)
+  rig.leftEye ? rig.leftEye?.node.getWorldPosition(eyePos) : eyePos.copy(headPos)
+
+  const avatarComponent = getMutableComponent(entity, AvatarComponent)
+  avatarComponent.torsoLength.set(Math.abs(headPos.y - hipsPos.y))
+  avatarComponent.upperLegLength.set(Math.abs(hipsPos.y - leftLowerLegPos.y))
+  avatarComponent.lowerLegLength.set(Math.abs(leftFootPos.y - leftUpperLegPos.y))
+  avatarComponent.hipsHeight.set(hipsPos.y)
+  avatarComponent.eyeHeight.set(eyePos.y)
+  avatarComponent.footGap.set(footGap.subVectors(leftFootPos, rightFootPos).length())
 
   computeTransformMatrix(entity)
 
@@ -220,29 +245,6 @@ export const setAvatarSpeedFromRootMotion = () => {
   if (walk) movement.walkSpeed.set(getRootSpeed(walk))
 }
 
-export const rigAvatarModel = (entity: Entity) => (model: VRM) => {
-  const avatarAnimationComponent = getComponent(entity, AvatarAnimationComponent)
-
-  const skinnedMeshes = findSkinnedMeshes(model.scene)
-
-  setComponent(entity, AvatarRigComponent, {
-    normalizedRig: model.humanoid.normalizedHumanBones,
-    rawRig: model.humanoid.rawHumanBones,
-    skinnedMeshes
-  })
-
-  avatarAnimationComponent.rootYRatio = 1
-
-  return model
-}
-
-export const setupAvatarHeight = (entity: Entity, model: Object3D) => {
-  const box = new Box3()
-  box.expandByObject(model).getSize(tempVec3ForHeight)
-  box.getCenter(tempVec3ForCenter)
-  resizeAvatar(entity, tempVec3ForHeight.y, tempVec3ForCenter)
-}
-
 export const getAvatarBoneWorldPosition = (entity: Entity, boneName: string, position: Vector3): boolean => {
   const avatarRigComponent = getOptionalComponent(entity, AvatarRigComponent)
   if (!avatarRigComponent || !avatarRigComponent.normalizedRig) return false