Skip to content

Commit

Permalink
When the local user is speaking, increase face tracking send rate:
Browse files Browse the repository at this point in the history 
- When the local user is speaking, increase the face tracking packet rate to 33 packets per second instead of 10 packets per second.
- This is experimental to see if better mouth movements could be captured.
  • Loading branch information
@hai-vr
hai-vr committed Dec 20, 2024
1 parent 31def36 commit b0df6d9
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 40 deletions.
59 changes: 42 additions & 17 deletions Basis/Packages/HVRBasisAvatarComms/Scripts/Actuation/BlendshapeActuation.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,20 @@ public class BlendshapeActuation : MonoBehaviour, ICommsNetworkable
{
private const int MaxAddresses = 256;
private const float BlendshapeAtFullStrength = 100f;
private const bool IsVoiceRelated = true;

[SerializeField] private SkinnedMeshRenderer[] renderers = Array.Empty<SkinnedMeshRenderer>();
[SerializeField] private BlendshapeActuationDefinitionFile[] definitionFiles = Array.Empty<BlendshapeActuationDefinitionFile>();
[SerializeField] private BlendshapeActuationDefinition[] definitions = Array.Empty<BlendshapeActuationDefinition>();

[HideInInspector] [SerializeField] private BasisAvatar avatar;
[HideInInspector] [SerializeField] private FeatureNetworking featureNetworking;
[HideInInspector] [SerializeField] private AcquisitionService acquisition;

private Dictionary<string, int> _addressBase = new Dictionary<string, int>();
private ComputedActuator[] _computedActuators;
private ComputedActuator[][] _addressBaseIndexToActuators;

#region NetworkingFields
private int _guidIndex;
// Can be null due to:
Expand All @@ -35,6 +36,7 @@ public class BlendshapeActuation : MonoBehaviour, ICommsNetworkable
private bool _avatarReady;
private bool _networkReady;
private bool _dualInitialized;
private bool _isWearer;

#endregion

Expand All @@ -43,7 +45,7 @@ private void Awake()
if (avatar == null) avatar = CommsUtil.GetAvatar(this);
if (featureNetworking == null) featureNetworking = CommsUtil.FeatureNetworkingFromAvatar(avatar);
if (acquisition == null) acquisition = AcquisitionService.SceneInstance;

renderers = CommsUtil.SlowSanitizeEndUserProvidedObjectArray(renderers);
definitionFiles = CommsUtil.SlowSanitizeEndUserProvidedObjectArray(definitionFiles);
definitions = CommsUtil.SlowSanitizeEndUserProvidedStructArray(definitions);
Expand All @@ -55,12 +57,12 @@ private void Awake()
private void OnAddressUpdated(string address, float inRange)
{
if (!_addressBase.TryGetValue(address, out var index)) return;

// TODO: Might need to queue and delay this change so that it executes on the Update loop.

var actuatorsForThisAddress = _addressBaseIndexToActuators[index];
if (actuatorsForThisAddress == null) return; // There may be no actuator for an address when it does not exist in the renderers.

var lower = 0f;
var upper = 0f;
foreach (var actuator in actuatorsForThisAddress)
Expand All @@ -69,7 +71,7 @@ private void OnAddressUpdated(string address, float inRange)
lower = actuator.StreamedLower;
upper = actuator.StreamedUpper;
}

if (_featureInterpolator != null)
{
var streamed01 = Mathf.InverseLerp(lower, upper, inRange);
Expand All @@ -83,7 +85,7 @@ private void OnInterpolatedDataChanged(float[] current)
{
var streamed01 = current[actuator.AddressIndex];
var inRange = Mathf.Lerp(actuator.StreamedLower, actuator.StreamedUpper, streamed01);

Actuate(actuator, inRange);
}
}
Expand All @@ -98,7 +100,7 @@ private static void Actuate(ComputedActuator actuator, float inRange)
var outputWild = Mathf.Lerp(actuator.OutStart, actuator.OutEnd, intermediate01);
var output01 = Mathf.Clamp01(outputWild);
var output0100 = output01 * BlendshapeAtFullStrength;

foreach (var target in actuator.Targets)
{
foreach (var blendshapeIndex in target.BlendshapeIndices)
Expand All @@ -110,11 +112,12 @@ private static void Actuate(ComputedActuator actuator, float inRange)

private void OnAvatarReady(bool isWearer)
{
_isWearer = isWearer;
var allDefinitions = definitions
.Concat(definitionFiles.SelectMany(file => file.definitions))
.ToArray();
_addressBase = MakeIndexDictionary(allDefinitions.Select(definition => definition.address).Distinct().ToArray());

if (_addressBase.Count > MaxAddresses)
{
Debug.LogError($"Exceeded max {MaxAddresses} addresses allowed in an actuator.");
Expand Down Expand Up @@ -145,7 +148,7 @@ private void OnAvatarReady(bool isWearer)
.ToArray();
return (inValuesForThisAddress.Min(), inValuesForThisAddress.Max());
});

_computedActuators = allDefinitions.Select(definition =>
{
var actuatorTargets = ComputeTargets(smrToBlendshapeNames, definition.blendshapes, definition.onlyFirstMatch);
Expand Down Expand Up @@ -180,7 +183,7 @@ private void OnAvatarReady(bool isWearer)
{
_addressBaseIndexToActuators[computedActuator.Key] = computedActuator.ToArray();
}

if (isWearer)
{
acquisition.RegisterAddresses(_addressBase.Keys.ToArray(), OnAddressUpdated);
Expand All @@ -193,7 +196,7 @@ private void OnAvatarReady(bool isWearer)
public void OnGuidAssigned(int guidIndex, Guid guid)
{
_guidIndex = guidIndex;

_networkReady = true;
TryOnAvatarIsNetworkable();
}
Expand All @@ -215,10 +218,30 @@ private void OnAvatarFullyNetworkable()
// FIXME: We should be using the computed actuators instead of the address base, assuming that
// the list of blendshapes is the same local and remote (no local-only or remote-only blendshapes).
_featureInterpolator = featureNetworking.NewInterpolator(_guidIndex, _addressBase.Count, OnInterpolatedDataChanged);

// FIXME: Add default values in the blendshape actuation file
if (_addressBase.TryGetValue("FT/v2/EyeLidLeft", out var indexLeft)) _featureInterpolator.Store(indexLeft, 0.8f);
if (_addressBase.TryGetValue("FT/v2/EyeLidRight", out var indexRight)) _featureInterpolator.Store(indexRight, 0.8f);

// TODO: Only enable these if the blendshape actuation is voice-related
if (_isWearer && IsVoiceRelated)
{
MicrophoneRecorder.MainThreadOnHasAudio -= MicrophoneTransmitting;
MicrophoneRecorder.MainThreadOnHasAudio += MicrophoneTransmitting;

MicrophoneRecorder.MainThreadOnHasSilence -= MicrophoneNotTransmitting;
MicrophoneRecorder.MainThreadOnHasSilence += MicrophoneNotTransmitting;
}
}

private void MicrophoneTransmitting()
{
_featureInterpolator.SwitchToHighSpeedTransmission();
}

private void MicrophoneNotTransmitting()
{
_featureInterpolator.SwitchToRegularSpeedTransmission();
}

private Dictionary<string, int> MakeIndexDictionary(string[] addressBase)
Expand All @@ -244,7 +267,9 @@ private void OnDisable()
private void OnDestroy()
{
avatar.OnAvatarReady -= OnAvatarReady;

MicrophoneRecorder.MainThreadOnHasAudio -= MicrophoneTransmitting;
MicrophoneRecorder.MainThreadOnHasSilence -= MicrophoneNotTransmitting;

acquisition.UnregisterAddresses(_addressBase.Keys.ToArray(), OnAddressUpdated);

if (_featureInterpolator != null)
Expand Down Expand Up @@ -286,7 +311,7 @@ private ComputedActuatorTarget[] ComputeTargets(Dictionary<SkinnedMeshRenderer,
.Select(toFind => pair.Value.IndexOf(toFind))
.Where(i => i >= 0)
.ToArray();

if (indices.Length > 0)
{
if (onlyFirstMatch)
Expand Down Expand Up @@ -332,4 +357,4 @@ private class ComputedActuatorTarget
public int[] BlendshapeIndices;
}
}
}
}
32 changes: 21 additions & 11 deletions Basis/Packages/HVRBasisAvatarComms/Scripts/FeatureNetworking/FeatureNetworking.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public class FeatureNetworking : MonoBehaviour
{
public const byte NegotiationPacket = 255;
public const byte ReservedPacket = 254;

public const byte ReservedPacket_RemoteRequestsInitializationMessage = 0;

public delegate void InterpolatedDataChanged(float[] current);
Expand All @@ -26,7 +26,7 @@ public class FeatureNetworking : MonoBehaviour
private Dictionary<Guid, ICommsNetworkable> _guidToNetworkable;
private Guid[] _orderedGuids;
private byte[] _negotiationPacket;

private IFeatureReceiver[] _featureHandles; // May contain null values if the corresponding Feature fails to initialize. Iterate defensively
private GameObject _holder;
private bool _isWearer;
Expand All @@ -39,7 +39,7 @@ private void Awake()
{
avatar.gameObject.AddComponent<HVRAvatarComms>();
}

var rand = new System.Random();
var safeNetPairings = netPairings
.Where(pairing => Guid.TryParse(pairing.guid, out _))
Expand Down Expand Up @@ -69,7 +69,7 @@ private void Awake()
// The order of the list of pairings should not matter between clients because of the Negotiation packet.
.OrderBy(_ => rand.Next())
.ToArray();

_guidToNetworkable = safeNetPairings.ToDictionary(pairing => new Guid(pairing.guid), pairing => (ICommsNetworkable)pairing.component);
_orderedGuids = safeNetPairings.Select(pairing => new Guid(pairing.guid)).ToArray();
_negotiationPacket = new [] { NegotiationPacket }
Expand Down Expand Up @@ -105,7 +105,7 @@ public FeatureInterpolator NewInterpolator(int guidIndex, int count, Interpolate
streamed.avatar = avatar;
streamed.valueArraySize = (byte)count; // TODO: Sanitize count to be within bounds
_holder.SetActive(true);

var handle = new FeatureInterpolator(this, guidIndex, streamed, interpolatedDataChanged);
streamed.OnInterpolatedDataChanged += handle.OnInterpolatedDataChanged;
streamed.SetEncodingInfo(_isWearer, (byte)guidIndex); // TODO: Make sure upstream that guidIndex is within limits
Expand Down Expand Up @@ -143,7 +143,7 @@ public void AssignGuids(bool isWearer)
{
var guid = _orderedGuids[index];
var networkable = _guidToNetworkable[guid];

networkable.OnGuidAssigned(index, guid);
}
}
Expand Down Expand Up @@ -192,7 +192,7 @@ public void TryResyncSome(ushort[] whoAsked)
public class FeatureEvent : IFeatureReceiver
{
private DeliveryMethod DeliveryMethod = DeliveryMethod.Sequenced;

private readonly FeatureNetworking _featureNetworking;
private readonly int _guidIndex;
private readonly FeatureNetworking.EventReceived _eventReceived;
Expand Down Expand Up @@ -239,17 +239,17 @@ public void Submit(ArraySegment<byte> currentState, ushort[] whoAsked)
{
if (whoAsked == null) throw new ArgumentException("whoAsked cannot be null");
if (whoAsked.Length == 0) throw new ArgumentException("whoAsked cannot be empty");

SubmitInternal(currentState, whoAsked);
}

private void SubmitInternal(ArraySegment<byte> currentState, ushort[] whoAskedNullable)
{
var buffer = new byte[1 + currentState.Count];
buffer[0] = (byte)_guidIndex;

currentState.CopyTo(buffer, 1);

_avatar.NetworkMessageSend(HVRAvatarComms.OurMessageIndex, buffer, DeliveryMethod, whoAskedNullable);
}
}
Expand Down Expand Up @@ -298,6 +298,16 @@ public void OnInterpolatedDataChanged(float[] current)
{
_interpolatedDataChanged.Invoke(current);
}

public void SwitchToHighSpeedTransmission()
{
_streamed.SwitchToHighSpeedTransmission();
}

public void SwitchToRegularSpeedTransmission()
{
_streamed.SwitchToRegularSpeedTransmission();
}
}

[Serializable]
Expand All @@ -313,4 +323,4 @@ public class RequestedFeature
public float lower;
public float upper;
}
}
}
Loading

0 comments on commit b0df6d9

Please sign in to comment.