Skip to content

Commit

Permalink
Add support for "select" voice command with OpenXR (#10661)
Browse files Browse the repository at this point in the history
* Add support for "select" voice command with OpenXR

* Update MicrosoftOpenXRGGVHand.cs
  • Loading branch information
MaxWang-MS authored Jun 24, 2022
1 parent d092fa9 commit a839860
Show file tree
Hide file tree
Showing 3 changed files with 176 additions and 0 deletions.
65 changes: 65 additions & 0 deletions Assets/MRTK/Providers/OpenXR/Scripts/MicrosoftOpenXRGGVHand.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

using Microsoft.MixedReality.Toolkit.Input;
using Microsoft.MixedReality.Toolkit.Utilities;
using Microsoft.MixedReality.Toolkit.XRSDK.Input;
using UnityEngine;

namespace Microsoft.MixedReality.Toolkit.XRSDK.OpenXR
{
/// <summary>
/// A GGV (Gaze, Gesture, and Voice) hand instance for OpenXR.
/// Used only for the purposes of acting on the select keyword detected by HoloLens 2.
/// </summary>
[MixedRealityController(
SupportedControllerType.GGVHand,
new[] { Handedness.Left, Handedness.Right, Handedness.None })]
internal class MicrosoftOpenXRGGVHand : GenericXRSDKController
{
public MicrosoftOpenXRGGVHand(
TrackingState trackingState,
Handedness controllerHandedness,
IMixedRealityInputSource inputSource = null,
MixedRealityInteractionMapping[] interactions = null)
: base(trackingState, controllerHandedness, inputSource, interactions, new SimpleHandDefinition(controllerHandedness))
{ }

internal void UpdateVoiceState(bool isPressed)
{
MixedRealityInteractionMapping interactionMapping = null;

for (int i = 0; i < Interactions?.Length; i++)
{
MixedRealityInteractionMapping currentInteractionMapping = Interactions[i];

if (currentInteractionMapping.AxisType == AxisType.Digital && currentInteractionMapping.InputType == DeviceInputType.Select)
{
interactionMapping = currentInteractionMapping;
break;
}
}

if (interactionMapping == null)
{
return;
}

interactionMapping.BoolData = isPressed;

// If our value changed raise it.
if (interactionMapping.Changed)
{
// Raise input system event if it's enabled
if (interactionMapping.BoolData)
{
CoreServices.InputSystem?.RaiseOnInputDown(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction);
}
else
{
CoreServices.InputSystem?.RaiseOnInputUp(InputSource, ControllerHandedness, interactionMapping.MixedRealityInputAction);
}
}
}
}
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

100 changes: 100 additions & 0 deletions Assets/MRTK/Providers/OpenXR/Scripts/OpenXRDeviceManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
#if MSFT_OPENXR && WINDOWS_UWP
using Microsoft.MixedReality.OpenXR;
using Microsoft.MixedReality.Toolkit.Windows.Input;
using Windows.UI.Input.Spatial;
#endif // MSFT_OPENXR && WINDOWS_UWP

namespace Microsoft.MixedReality.Toolkit.XRSDK.OpenXR
Expand Down Expand Up @@ -80,6 +81,7 @@ public override void Enable()

#if MSFT_OPENXR && WINDOWS_UWP
CreateGestureRecognizers();
SpatialInteractionManager.SourcePressed += SpatialInteractionManager_SourcePressed;
#endif // MSFT_OPENXR && WINDOWS_UWP

base.Enable();
Expand Down Expand Up @@ -114,6 +116,26 @@ public override void Update()
base.Update();

CheckForGestures();

if (shouldSendVoiceEvents)
{
MicrosoftOpenXRGGVHand controller = GetOrAddVoiceController();
if (controller != null)
{
// RaiseOnInputDown for "select"
controller.UpdateVoiceState(true);
// RaiseOnInputUp for "select"
controller.UpdateVoiceState(false);

// On WMR, the voice recognizer does not actually register the phrase 'select'
// when you add it to the speech commands profile. Therefore, simulate
// the "select" voice command running to ensure that we get a select voice command
// registered. This is used by FocusProvider to detect when the select pointer is active.
Service?.RaiseSpeechCommandRecognized(controller.InputSource, RecognitionConfidenceLevel.High, TimeSpan.MinValue, DateTime.Now, new SpeechCommands("select", KeyCode.Alpha1, MixedRealityInputAction.None));
}

shouldSendVoiceEvents = false;
}
}

/// <inheritdoc />
Expand All @@ -140,6 +162,14 @@ public override void Disable()
#endif
navigationGestureRecognizer = null;

SpatialInteractionManager.SourcePressed -= SpatialInteractionManager_SourcePressed;

if (voiceController != null)
{
RemoveControllerFromScene(voiceController);
voiceController = null;
}

base.Disable();
}
#endif // MSFT_OPENXR && WINDOWS_UWP
Expand Down Expand Up @@ -529,5 +559,75 @@ private GenericXRSDKController FindMatchingController(GestureHandedness gestureH
#endif // MSFT_OPENXR && WINDOWS_UWP

#endregion Gesture implementation

#region SpatialInteractionManager event and helpers

#if MSFT_OPENXR && WINDOWS_UWP
/// <summary>
/// SDK Interaction Source Pressed Event handler. Used only for voice.
/// </summary>
/// <param name="args">SDK source pressed event arguments</param>
private void SpatialInteractionManager_SourcePressed(SpatialInteractionManager sender, SpatialInteractionSourceEventArgs args)
{
if (args.State.Source.Kind == SpatialInteractionSourceKind.Voice)
{
shouldSendVoiceEvents = true;
}
}

private MicrosoftOpenXRGGVHand voiceController = null;
private bool shouldSendVoiceEvents = false;

private MicrosoftOpenXRGGVHand GetOrAddVoiceController()
{
if (voiceController != null)
{
return voiceController;
}

IMixedRealityInputSource inputSource = Service?.RequestNewGenericInputSource("Mixed Reality Voice", sourceType: InputSourceType.Voice);
MicrosoftOpenXRGGVHand detectedController = new MicrosoftOpenXRGGVHand(TrackingState.NotTracked, Utilities.Handedness.None, inputSource);

if (!detectedController.Enabled)
{
// Controller failed to be setup correctly.
// Return null so we don't raise the source detected.
return null;
}

for (int i = 0; i < detectedController.InputSource?.Pointers?.Length; i++)
{
detectedController.InputSource.Pointers[i].Controller = detectedController;
}

Service?.RaiseSourceDetected(detectedController.InputSource, detectedController);

voiceController = detectedController;
return voiceController;
}

private SpatialInteractionManager spatialInteractionManager = null;

/// <summary>
/// Provides access to the current native SpatialInteractionManager.
/// </summary>
private SpatialInteractionManager SpatialInteractionManager
{
get
{
if (spatialInteractionManager == null)
{
UnityEngine.WSA.Application.InvokeOnUIThread(() =>
{
spatialInteractionManager = SpatialInteractionManager.GetForCurrentView();
}, true);
}

return spatialInteractionManager;
}
}
#endif // MSFT_OPENXR && WINDOWS_UWP

#endregion SpatialInteractionManager events
}
}

0 comments on commit a839860

Please sign in to comment.