Improve spatial input

packages/client-core/src/common/components/TouchGamepad/index.tsx

@@ -28,7 +28,16 @@ import React from 'react'
 import { Joystick } from 'react-joystick-component'
 
 import { isTouchAvailable } from '@etherealengine/engine/src/common/functions/DetectFeatures'
-import { OldButtonTypes } from '@etherealengine/engine/src/input/InputState'
+import { getComponent } from '@etherealengine/engine/src/ecs/functions/ComponentFunctions'
+import {
+  getFirstNonCapturedInputSource,
+  InputSourceComponent
+} from '@etherealengine/engine/src/input/components/InputSourceComponent'
+import {
+  AnyButton,
+  createInitialButtonState,
+  XRStandardGamepadButton
+} from '@etherealengine/engine/src/input/state/ButtonState'
 import { InteractState } from '@etherealengine/engine/src/interaction/systems/InteractiveSystem'
 import { isMobileXRHeadset } from '@etherealengine/engine/src/xr/XRState'
 import { getMutableState } from '@etherealengine/hyperflux'
@@ -37,7 +46,17 @@ import Icon from '@etherealengine/ui/src/primitives/mui/Icon'
 import { AppState } from '../../services/AppService'
 import styles from './index.module.scss'
 
-const triggerButton = (button: OldButtonTypes, pressed: boolean): void => {
+const triggerButton = (button: AnyButton, pressed: boolean): void => {
+  const nonCapturedInputSource = getFirstNonCapturedInputSource()
+  if (!nonCapturedInputSource) return
+
+  const inputSource = getComponent(nonCapturedInputSource, InputSourceComponent)
+
+  let buttonState = inputSource.buttons[button]
+  if (buttonState || pressed) {
+    buttonState = buttonState || createInitialButtonState()
+    buttonState.pressed = pressed
+  }
   const eventType = pressed ? 'touchgamepadbuttondown' : 'touchgamepadbuttonup'
   const event = new CustomEvent(eventType, { detail: { button } })
   document.dispatchEvent(event)
@@ -72,10 +91,10 @@ const handleStop = () => {
   document.dispatchEvent(event)
 }
 
-const buttonsConfig: Array<{ button: OldButtonTypes; label: string }> = [
+const buttonsConfig: Array<{ button: AnyButton; label: React.ReactElement }> = [
   {
-    button: 'ButtonA',
-    label: 'A'
+    button: XRStandardGamepadButton.Trigger,
+    label: <Icon type="TouchApp" />
   }
 ]
 
@@ -94,7 +113,7 @@ export const TouchGamepad = () => {
         onPointerDown={(): void => triggerButton(value.button, true)}
         onPointerUp={(): void => triggerButton(value.button, false)}
       >
-        <Icon type="TouchApp" />
+        {value.label}
       </div>
     )
   })

packages/client-core/src/systems/AvatarUISystem.tsx

@@ -38,6 +38,10 @@ import { Entity } from '@etherealengine/engine/src/ecs/classes/Entity'
 import { defineQuery, getComponent, hasComponent } from '@etherealengine/engine/src/ecs/functions/ComponentFunctions'
 import { removeEntity } from '@etherealengine/engine/src/ecs/functions/EntityFunctions'
 import { defineSystem } from '@etherealengine/engine/src/ecs/functions/SystemFunctions'
+import {
+  getFirstNonCapturedInputSource,
+  InputSourceComponent
+} from '@etherealengine/engine/src/input/components/InputSourceComponent'
 import { NetworkObjectComponent } from '@etherealengine/engine/src/networking/components/NetworkObjectComponent'
 import { NetworkObjectOwnedTag } from '@etherealengine/engine/src/networking/components/NetworkObjectComponent'
 import { MediaSettingsState } from '@etherealengine/engine/src/networking/MediaSettingsState'
@@ -147,12 +151,17 @@ const execute = () => {
   const engineState = getState(EngineState)
   if (!engineState.isEngineInitialized) return
 
-  const keys = Engine.instance.buttons
+  const nonCapturedInputSource = getFirstNonCapturedInputSource()
+  if (!nonCapturedInputSource) return
+
+  const inputSource = getComponent(nonCapturedInputSource, InputSourceComponent)
+
+  const keys = inputSource.buttons
 
   if (keys.PrimaryClick?.down) onPrimaryClick()
   if (keys.SecondaryClick?.down) onSecondaryClick()
 
-  videoPreviewTimer += Engine.instance.deltaSeconds
+  videoPreviewTimer += engineState.deltaSeconds
   if (videoPreviewTimer > 1) videoPreviewTimer = 0
 
   for (const userEntity of userQuery.enter()) {

packages/client-core/src/systems/WidgetUISystem.ts

@@ -31,13 +31,16 @@ import { V_001, V_010, V_111 } from '@etherealengine/engine/src/common/constants
 import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine'
 import {
   addComponent,
+  defineQuery,
   getComponent,
   hasComponent,
   removeComponent,
   setComponent
 } from '@etherealengine/engine/src/ecs/functions/ComponentFunctions'
 import { removeEntity } from '@etherealengine/engine/src/ecs/functions/EntityFunctions'
 import { defineSystem } from '@etherealengine/engine/src/ecs/functions/SystemFunctions'
+import { InputSourceComponent } from '@etherealengine/engine/src/input/components/InputSourceComponent'
+import { StandardGamepadButton, XRStandardGamepadButton } from '@etherealengine/engine/src/input/state/ButtonState'
 import { addObjectToGroup } from '@etherealengine/engine/src/scene/components/GroupComponent'
 import { NameComponent } from '@etherealengine/engine/src/scene/components/NameComponent'
 import { setVisibleComponent, VisibleComponent } from '@etherealengine/engine/src/scene/components/VisibleComponent'
@@ -154,19 +157,27 @@ const toggleWidgetsMenu = (handedness?: 'left' | 'right') => {
   }
 }
 
+const inputSourceQuery = defineQuery([InputSourceComponent])
+
 const showWidgetQueue = defineActionQueue(WidgetAppActions.showWidget.matches)
 const registerWidgetQueue = defineActionQueue(WidgetAppActions.registerWidget.matches)
 const unregisterWidgetQueue = defineActionQueue(WidgetAppActions.unregisterWidget.matches)
 
 const execute = () => {
   const widgetState = getState(WidgetAppState)
   const { widgetMenuUI } = getState(WidgetUISystemState)
+  const inputSources = inputSourceQuery()
 
-  const keys = Engine.instance.buttons
-  if (keys.ButtonX?.down) toggleWidgetsMenu('left')
-  if (keys.ButtonA?.down) toggleWidgetsMenu('right')
-  /** @todo allow non HMDs to access the widget menu too */
-  if ((isDev || isMobileXRHeadset) && keys.Escape?.down) toggleWidgetsMenu()
+  for (const inputSourceEntity of inputSources) {
+    const inputSource = getComponent(inputSourceEntity, InputSourceComponent)
+    const keys = inputSource.buttons
+    if (inputSource.source.gamepad?.mapping === 'xr-standard') {
+      if (keys[XRStandardGamepadButton.ButtonA]?.down)
+        toggleWidgetsMenu(inputSource.source.handedness === 'left' ? 'right' : 'left')
+    }
+    /** @todo allow non HMDs to access the widget menu too */
+    if ((isDev || isMobileXRHeadset) && keys.Escape?.down) toggleWidgetsMenu()
+  }
 
   for (const action of showWidgetQueue()) {
     const widget = Engine.instance.widgets.get(action.id)!
@@ -186,11 +197,12 @@ const execute = () => {
   }
 
   const transform = getComponent(widgetMenuUI.entity, TransformComponent)
-  const activeInputSource = Array.from(Engine.instance.inputSources).find(
-    (inputSource) => inputSource.handedness === widgetState.handedness
+  const activeInputSourceEntity = inputSources.find(
+    (entity) => getComponent(entity, InputSourceComponent).source.handedness === widgetState.handedness
   )
 
-  if (activeInputSource) {
+  if (activeInputSourceEntity) {
+    const activeInputSource = getComponent(activeInputSourceEntity, InputSourceComponent)?.source
     const referenceSpace = ReferenceSpace.origin!
     const pose = Engine.instance.xrFrame!.getPose(
       activeInputSource.gripSpace ?? activeInputSource.targetRaySpace,

packages/editor/src/systems/EditorControlSystem.ts

@@ -59,6 +59,10 @@ import {
   getEntityNodeArrayFromEntities
 } from '@etherealengine/engine/src/ecs/functions/EntityTree'
 import { defineSystem } from '@etherealengine/engine/src/ecs/functions/SystemFunctions'
+import {
+  getFirstNonCapturedInputSource,
+  InputSourceComponent
+} from '@etherealengine/engine/src/input/components/InputSourceComponent'
 import InfiniteGridHelper from '@etherealengine/engine/src/scene/classes/InfiniteGridHelper'
 import { addObjectToGroup, GroupComponent } from '@etherealengine/engine/src/scene/components/GroupComponent'
 import { NameComponent } from '@etherealengine/engine/src/scene/components/NameComponent'
@@ -225,9 +229,9 @@ const onKeyX = () => {
   toggleTransformPivot()
 }
 
-const onKeyZ = () => {
-  if (Engine.instance.buttons.ControlLeft?.pressed) {
-    if (Engine.instance.buttons.ShiftLeft?.pressed) {
+const onKeyZ = (control: boolean, shift: boolean) => {
+  if (control) {
+    if (shift) {
       dispatchAction(EditorHistoryAction.redo({ count: 1 }))
     } else {
       dispatchAction(EditorHistoryAction.undo({ count: 1 }))
@@ -374,8 +378,6 @@ const execute = () => {
   transformSpaceChanged = transformSpace !== editorHelperState.transformSpace
   transformSpace = editorHelperState.transformSpace
 
-  const inputState = Engine.instance.buttons
-
   if (selectedParentEntities.length === 0 || transformMode === TransformMode.Disabled) {
     if (hasComponent(gizmoEntity, VisibleComponent)) removeComponent(gizmoEntity, VisibleComponent)
   } else {
@@ -438,10 +440,16 @@ const execute = () => {
   }
   const cursorPosition = Engine.instance.pointerState.position
 
+  const nonCapturedInputSource = getFirstNonCapturedInputSource()
+  if (!nonCapturedInputSource) return
+
+  const inputSource = getComponent(nonCapturedInputSource, InputSourceComponent)
+  const buttons = inputSource.buttons
+
   const isGrabbing = transformMode === TransformMode.Grab || transformMode === TransformMode.Placement
 
-  const isPrimaryClickDown = inputState.PrimaryClick?.down
-  const isPrimaryClickUp = inputState.PrimaryClick?.up
+  const isPrimaryClickDown = buttons.PrimaryClick?.down
+  const isPrimaryClickUp = buttons.PrimaryClick?.up
 
   const selectStartAndNoGrabbing = isPrimaryClickDown && !isGrabbing
 
@@ -463,7 +471,7 @@ const execute = () => {
     gizmoObj.highlightHoveredAxis(cursorPosition)
   }
 
-  const modifier = isMacOS ? inputState.MetaLeft?.pressed : inputState.ControlLeft?.pressed
+  const modifier = isMacOS ? buttons.MetaLeft?.pressed : buttons.ControlLeft?.pressed
   const shouldSnap = (editorHelperState.snapMode === SnapMode.Grid) === !modifier
 
   if (dragging || isGrabbing) {
@@ -630,7 +638,7 @@ const execute = () => {
   }
 
   selectionCounter = selectionState.selectionCounter
-  const shift = inputState.ShiftLeft?.pressed
+  const shift = buttons.ShiftLeft?.pressed
 
   if (isPrimaryClickUp) {
     if (transformMode === TransformMode.Grab) {
@@ -664,24 +672,24 @@ const execute = () => {
 
   if (editorHelperState.isFlyModeEnabled) return
 
-  if (inputState.KeyQ?.down) onKeyQ()
-  if (inputState.KeyE?.down) onKeyE()
-  if (inputState.KeyG?.down) onKeyG()
-  if (inputState.Escape?.down) onEscape()
-  if (inputState.KeyF?.down) onKeyF()
-  if (inputState.KeyT?.down) onKeyT()
-  if (inputState.KeyR?.down) onKeyR()
-  if (inputState.KeyY?.down) onKeyY()
-  if (inputState.KeyC?.down) onKeyC()
-  if (inputState.KeyX?.down) onKeyX()
-  if (inputState.KeyZ?.down) onKeyZ()
-  if (inputState.Equal?.down) onEqual()
-  if (inputState.Minus?.down) onMinus()
-  if (inputState.Delete?.down) onDelete()
-
-  const selecting = inputState.PrimaryClick?.pressed && !dragging
+  if (buttons.KeyQ?.down) onKeyQ()
+  if (buttons.KeyE?.down) onKeyE()
+  if (buttons.KeyG?.down) onKeyG()
+  if (buttons.Escape?.down) onEscape()
+  if (buttons.KeyF?.down) onKeyF()
+  if (buttons.KeyT?.down) onKeyT()
+  if (buttons.KeyR?.down) onKeyR()
+  if (buttons.KeyY?.down) onKeyY()
+  if (buttons.KeyC?.down) onKeyC()
+  if (buttons.KeyX?.down) onKeyX()
+  if (buttons.KeyZ?.down) onKeyZ(!!buttons.ControlLeft?.pressed, !!buttons.ShiftLeft?.pressed)
+  if (buttons.Equal?.down) onEqual()
+  if (buttons.Minus?.down) onMinus()
+  if (buttons.Delete?.down) onDelete()
+
+  const selecting = buttons.PrimaryClick?.pressed && !dragging
   const zoom = Engine.instance.pointerState.scroll.y
-  const panning = inputState.AuxiliaryClick?.pressed
+  const panning = buttons.AuxiliaryClick?.pressed
 
   if (selecting) {
     const editorCameraState = getMutableState(EditorCameraState)

packages/editor/src/systems/EditorFlyControlSystem.ts

@@ -28,11 +28,16 @@ import { MathUtils, Matrix3, Vector3 } from 'three'
 import { FlyControlComponent } from '@etherealengine/engine/src/avatar/components/FlyControlComponent'
 import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine'
 import {
+  getComponent,
   hasComponent,
   removeComponent,
   setComponent
 } from '@etherealengine/engine/src/ecs/functions/ComponentFunctions'
 import { defineSystem } from '@etherealengine/engine/src/ecs/functions/SystemFunctions'
+import {
+  getFirstNonCapturedInputSource,
+  InputSourceComponent
+} from '@etherealengine/engine/src/input/components/InputSourceComponent'
 import { dispatchAction } from '@etherealengine/hyperflux'
 
 import { editorCameraCenter } from '../classes/EditorCameraState'
@@ -67,9 +72,13 @@ const onSecondaryReleased = () => {
 }
 
 const execute = () => {
-  const keys = Engine.instance.buttons
-  if (keys.SecondaryClick?.down) onSecondaryClick()
-  if (keys.SecondaryClick?.up) onSecondaryReleased()
+  const nonCapturedInputSource = getFirstNonCapturedInputSource()
+  if (!nonCapturedInputSource) return
+
+  const inputSource = getComponent(nonCapturedInputSource, InputSourceComponent)
+
+  if (inputSource.buttons.SecondaryClick?.down) onSecondaryClick()
+  if (inputSource.buttons.SecondaryClick?.up) onSecondaryReleased()
 }
 
 export const EditorFlyControlSystem = defineSystem({

packages/engine/src/audio/systems/PositionalAudioSystem.tsx

@@ -35,8 +35,7 @@ import { Engine } from '../../ecs/classes/Engine'
 import { ComponentType, defineQuery, getComponent, useComponent } from '../../ecs/functions/ComponentFunctions'
 import { useEntityContext } from '../../ecs/functions/EntityFunctions'
 import { createQueryReactor, defineSystem } from '../../ecs/functions/SystemFunctions'
-import { LocalAvatarTagComponent } from '../../input/components/LocalAvatarTagComponent'
-import { NetworkObjectComponent } from '../../networking/components/NetworkObjectComponent'
+import { NetworkObjectComponent, NetworkObjectOwnedTag } from '../../networking/components/NetworkObjectComponent'
 import { MediaSettingsState } from '../../networking/MediaSettingsState'
 import { webcamAudioDataChannelType } from '../../networking/NetworkState'
 import { AudioNodeGroups, createAudioNodeGroup, MediaElementComponent } from '../../scene/components/MediaComponent'
@@ -56,7 +55,7 @@ const positionalAudioQuery = defineQuery([PositionalAudioComponent, MediaElement
 /**
  * Avatars
  */
-const networkedAvatarAudioQuery = defineQuery([AvatarComponent, NetworkObjectComponent, Not(LocalAvatarTagComponent)])
+const networkedAvatarAudioQuery = defineQuery([AvatarComponent, NetworkObjectComponent, Not(NetworkObjectOwnedTag)])
 
 const setMediaStreamVolumeActionQueue = defineActionQueue(AudioSettingAction.setMediaStreamVolume.matches)
 

packages/engine/src/avatar/AvatarAnimationSystem.ts

@@ -40,6 +40,7 @@ import { defineQuery, getComponent, getOptionalComponent, setComponent } from '.
 import { removeEntity } from '../ecs/functions/EntityFunctions'
 import { defineSystem } from '../ecs/functions/SystemFunctions'
 import { createPriorityQueue } from '../ecs/PriorityQueue'
+import { InputSourceComponent } from '../input/components/InputSourceComponent'
 import { NetworkObjectComponent } from '../networking/components/NetworkObjectComponent'
 import { RigidBodyComponent } from '../physics/components/RigidBodyComponent'
 import { addObjectToGroup, GroupComponent } from '../scene/components/GroupComponent'
@@ -115,6 +116,8 @@ const sessionChangedQueue = defineActionQueue(XRAction.sessionChanged.matches)
 
 const ikTargetQuery = defineQuery([AvatarIKTargetComponent])
 
+const inputSourceQuery = defineQuery([InputSourceComponent])
+
 const minimumFrustumCullDistanceSqr = 5 * 5 // 5 units
 
 const filterPriorityEntities = (entity: Entity) =>
@@ -138,7 +141,7 @@ const _quat = new Quaternion()
 const execute = () => {
   const xrState = getState(XRState)
   const { priorityQueue, sortedTransformEntities } = getState(AvatarAnimationState)
-  const { localClientEntity, inputSources } = Engine.instance
+  const { localClientEntity } = Engine.instance
   const { elapsedSeconds, deltaSeconds } = getState(EngineState)
 
   for (const action of sessionChangedQueue()) {
@@ -173,7 +176,8 @@ const execute = () => {
 
   // todo - remove ik targets when session ends
   if (xrState.sessionActive && localClientEntity) {
-    const sources = Array.from(inputSources.values())
+    const sources = inputSourceQuery().map((eid) => getComponent(eid, InputSourceComponent).source)
+
     const head = getCameraMode() === 'attached'
     const leftHand = !!sources.find((s) => s.handedness === 'left')
     const rightHand = !!sources.find((s) => s.handedness === 'right')