From b0df6d9ba16f848169e3e34dc4bf1b5097db4471 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ha=C3=AF=7E?= Date: Fri, 20 Dec 2024 07:00:00 +0100 Subject: [PATCH] When the local user is speaking, increase face tracking send rate: - When the local user is speaking, increase the face tracking packet rate to 33 packets per second instead of 10 packets per second. - This is experimental to see if better mouth movements could be captured. --- .../Scripts/Actuation/BlendshapeActuation.cs | 59 +++++++++++++------ .../FeatureNetworking/FeatureNetworking.cs | 32 ++++++---- .../Networking/StreamedAvatarFeature.cs | 36 +++++++---- 3 files changed, 87 insertions(+), 40 deletions(-) diff --git a/Basis/Packages/HVRBasisAvatarComms/Scripts/Actuation/BlendshapeActuation.cs b/Basis/Packages/HVRBasisAvatarComms/Scripts/Actuation/BlendshapeActuation.cs index 9b6ecebd0..bd7cfd69a 100644 --- a/Basis/Packages/HVRBasisAvatarComms/Scripts/Actuation/BlendshapeActuation.cs +++ b/Basis/Packages/HVRBasisAvatarComms/Scripts/Actuation/BlendshapeActuation.cs @@ -11,19 +11,20 @@ public class BlendshapeActuation : MonoBehaviour, ICommsNetworkable { private const int MaxAddresses = 256; private const float BlendshapeAtFullStrength = 100f; + private const bool IsVoiceRelated = true; [SerializeField] private SkinnedMeshRenderer[] renderers = Array.Empty(); [SerializeField] private BlendshapeActuationDefinitionFile[] definitionFiles = Array.Empty(); [SerializeField] private BlendshapeActuationDefinition[] definitions = Array.Empty(); - + [HideInInspector] [SerializeField] private BasisAvatar avatar; [HideInInspector] [SerializeField] private FeatureNetworking featureNetworking; [HideInInspector] [SerializeField] private AcquisitionService acquisition; - + private Dictionary _addressBase = new Dictionary(); private ComputedActuator[] _computedActuators; private ComputedActuator[][] _addressBaseIndexToActuators; - + #region NetworkingFields private int _guidIndex; // Can be null due to: @@ -35,6 +36,7 @@ public class BlendshapeActuation : MonoBehaviour, ICommsNetworkable private bool _avatarReady; private bool _networkReady; private bool _dualInitialized; + private bool _isWearer; #endregion @@ -43,7 +45,7 @@ private void Awake() if (avatar == null) avatar = CommsUtil.GetAvatar(this); if (featureNetworking == null) featureNetworking = CommsUtil.FeatureNetworkingFromAvatar(avatar); if (acquisition == null) acquisition = AcquisitionService.SceneInstance; - + renderers = CommsUtil.SlowSanitizeEndUserProvidedObjectArray(renderers); definitionFiles = CommsUtil.SlowSanitizeEndUserProvidedObjectArray(definitionFiles); definitions = CommsUtil.SlowSanitizeEndUserProvidedStructArray(definitions); @@ -55,12 +57,12 @@ private void Awake() private void OnAddressUpdated(string address, float inRange) { if (!_addressBase.TryGetValue(address, out var index)) return; - + // TODO: Might need to queue and delay this change so that it executes on the Update loop. var actuatorsForThisAddress = _addressBaseIndexToActuators[index]; if (actuatorsForThisAddress == null) return; // There may be no actuator for an address when it does not exist in the renderers. - + var lower = 0f; var upper = 0f; foreach (var actuator in actuatorsForThisAddress) @@ -69,7 +71,7 @@ private void OnAddressUpdated(string address, float inRange) lower = actuator.StreamedLower; upper = actuator.StreamedUpper; } - + if (_featureInterpolator != null) { var streamed01 = Mathf.InverseLerp(lower, upper, inRange); @@ -83,7 +85,7 @@ private void OnInterpolatedDataChanged(float[] current) { var streamed01 = current[actuator.AddressIndex]; var inRange = Mathf.Lerp(actuator.StreamedLower, actuator.StreamedUpper, streamed01); - + Actuate(actuator, inRange); } } @@ -98,7 +100,7 @@ private static void Actuate(ComputedActuator actuator, float inRange) var outputWild = Mathf.Lerp(actuator.OutStart, actuator.OutEnd, intermediate01); var output01 = Mathf.Clamp01(outputWild); var output0100 = output01 * BlendshapeAtFullStrength; - + foreach (var target in actuator.Targets) { foreach (var blendshapeIndex in target.BlendshapeIndices) @@ -110,11 +112,12 @@ private static void Actuate(ComputedActuator actuator, float inRange) private void OnAvatarReady(bool isWearer) { + _isWearer = isWearer; var allDefinitions = definitions .Concat(definitionFiles.SelectMany(file => file.definitions)) .ToArray(); _addressBase = MakeIndexDictionary(allDefinitions.Select(definition => definition.address).Distinct().ToArray()); - + if (_addressBase.Count > MaxAddresses) { Debug.LogError($"Exceeded max {MaxAddresses} addresses allowed in an actuator."); @@ -145,7 +148,7 @@ private void OnAvatarReady(bool isWearer) .ToArray(); return (inValuesForThisAddress.Min(), inValuesForThisAddress.Max()); }); - + _computedActuators = allDefinitions.Select(definition => { var actuatorTargets = ComputeTargets(smrToBlendshapeNames, definition.blendshapes, definition.onlyFirstMatch); @@ -180,7 +183,7 @@ private void OnAvatarReady(bool isWearer) { _addressBaseIndexToActuators[computedActuator.Key] = computedActuator.ToArray(); } - + if (isWearer) { acquisition.RegisterAddresses(_addressBase.Keys.ToArray(), OnAddressUpdated); @@ -193,7 +196,7 @@ private void OnAvatarReady(bool isWearer) public void OnGuidAssigned(int guidIndex, Guid guid) { _guidIndex = guidIndex; - + _networkReady = true; TryOnAvatarIsNetworkable(); } @@ -215,10 +218,30 @@ private void OnAvatarFullyNetworkable() // FIXME: We should be using the computed actuators instead of the address base, assuming that // the list of blendshapes is the same local and remote (no local-only or remote-only blendshapes). _featureInterpolator = featureNetworking.NewInterpolator(_guidIndex, _addressBase.Count, OnInterpolatedDataChanged); - + // FIXME: Add default values in the blendshape actuation file if (_addressBase.TryGetValue("FT/v2/EyeLidLeft", out var indexLeft)) _featureInterpolator.Store(indexLeft, 0.8f); if (_addressBase.TryGetValue("FT/v2/EyeLidRight", out var indexRight)) _featureInterpolator.Store(indexRight, 0.8f); + + // TODO: Only enable these if the blendshape actuation is voice-related + if (_isWearer && IsVoiceRelated) + { + MicrophoneRecorder.MainThreadOnHasAudio -= MicrophoneTransmitting; + MicrophoneRecorder.MainThreadOnHasAudio += MicrophoneTransmitting; + + MicrophoneRecorder.MainThreadOnHasSilence -= MicrophoneNotTransmitting; + MicrophoneRecorder.MainThreadOnHasSilence += MicrophoneNotTransmitting; + } + } + + private void MicrophoneTransmitting() + { + _featureInterpolator.SwitchToHighSpeedTransmission(); + } + + private void MicrophoneNotTransmitting() + { + _featureInterpolator.SwitchToRegularSpeedTransmission(); } private Dictionary MakeIndexDictionary(string[] addressBase) @@ -244,7 +267,9 @@ private void OnDisable() private void OnDestroy() { avatar.OnAvatarReady -= OnAvatarReady; - + MicrophoneRecorder.MainThreadOnHasAudio -= MicrophoneTransmitting; + MicrophoneRecorder.MainThreadOnHasSilence -= MicrophoneNotTransmitting; + acquisition.UnregisterAddresses(_addressBase.Keys.ToArray(), OnAddressUpdated); if (_featureInterpolator != null) @@ -286,7 +311,7 @@ private ComputedActuatorTarget[] ComputeTargets(Dictionary pair.Value.IndexOf(toFind)) .Where(i => i >= 0) .ToArray(); - + if (indices.Length > 0) { if (onlyFirstMatch) @@ -332,4 +357,4 @@ private class ComputedActuatorTarget public int[] BlendshapeIndices; } } -} \ No newline at end of file +} diff --git a/Basis/Packages/HVRBasisAvatarComms/Scripts/FeatureNetworking/FeatureNetworking.cs b/Basis/Packages/HVRBasisAvatarComms/Scripts/FeatureNetworking/FeatureNetworking.cs index 85d613116..a83735c57 100644 --- a/Basis/Packages/HVRBasisAvatarComms/Scripts/FeatureNetworking/FeatureNetworking.cs +++ b/Basis/Packages/HVRBasisAvatarComms/Scripts/FeatureNetworking/FeatureNetworking.cs @@ -12,7 +12,7 @@ public class FeatureNetworking : MonoBehaviour { public const byte NegotiationPacket = 255; public const byte ReservedPacket = 254; - + public const byte ReservedPacket_RemoteRequestsInitializationMessage = 0; public delegate void InterpolatedDataChanged(float[] current); @@ -26,7 +26,7 @@ public class FeatureNetworking : MonoBehaviour private Dictionary _guidToNetworkable; private Guid[] _orderedGuids; private byte[] _negotiationPacket; - + private IFeatureReceiver[] _featureHandles; // May contain null values if the corresponding Feature fails to initialize. Iterate defensively private GameObject _holder; private bool _isWearer; @@ -39,7 +39,7 @@ private void Awake() { avatar.gameObject.AddComponent(); } - + var rand = new System.Random(); var safeNetPairings = netPairings .Where(pairing => Guid.TryParse(pairing.guid, out _)) @@ -69,7 +69,7 @@ private void Awake() // The order of the list of pairings should not matter between clients because of the Negotiation packet. .OrderBy(_ => rand.Next()) .ToArray(); - + _guidToNetworkable = safeNetPairings.ToDictionary(pairing => new Guid(pairing.guid), pairing => (ICommsNetworkable)pairing.component); _orderedGuids = safeNetPairings.Select(pairing => new Guid(pairing.guid)).ToArray(); _negotiationPacket = new [] { NegotiationPacket } @@ -105,7 +105,7 @@ public FeatureInterpolator NewInterpolator(int guidIndex, int count, Interpolate streamed.avatar = avatar; streamed.valueArraySize = (byte)count; // TODO: Sanitize count to be within bounds _holder.SetActive(true); - + var handle = new FeatureInterpolator(this, guidIndex, streamed, interpolatedDataChanged); streamed.OnInterpolatedDataChanged += handle.OnInterpolatedDataChanged; streamed.SetEncodingInfo(_isWearer, (byte)guidIndex); // TODO: Make sure upstream that guidIndex is within limits @@ -143,7 +143,7 @@ public void AssignGuids(bool isWearer) { var guid = _orderedGuids[index]; var networkable = _guidToNetworkable[guid]; - + networkable.OnGuidAssigned(index, guid); } } @@ -192,7 +192,7 @@ public void TryResyncSome(ushort[] whoAsked) public class FeatureEvent : IFeatureReceiver { private DeliveryMethod DeliveryMethod = DeliveryMethod.Sequenced; - + private readonly FeatureNetworking _featureNetworking; private readonly int _guidIndex; private readonly FeatureNetworking.EventReceived _eventReceived; @@ -239,7 +239,7 @@ public void Submit(ArraySegment currentState, ushort[] whoAsked) { if (whoAsked == null) throw new ArgumentException("whoAsked cannot be null"); if (whoAsked.Length == 0) throw new ArgumentException("whoAsked cannot be empty"); - + SubmitInternal(currentState, whoAsked); } @@ -247,9 +247,9 @@ private void SubmitInternal(ArraySegment currentState, ushort[] whoAskedNu { var buffer = new byte[1 + currentState.Count]; buffer[0] = (byte)_guidIndex; - + currentState.CopyTo(buffer, 1); - + _avatar.NetworkMessageSend(HVRAvatarComms.OurMessageIndex, buffer, DeliveryMethod, whoAskedNullable); } } @@ -298,6 +298,16 @@ public void OnInterpolatedDataChanged(float[] current) { _interpolatedDataChanged.Invoke(current); } + + public void SwitchToHighSpeedTransmission() + { + _streamed.SwitchToHighSpeedTransmission(); + } + + public void SwitchToRegularSpeedTransmission() + { + _streamed.SwitchToRegularSpeedTransmission(); + } } [Serializable] @@ -313,4 +323,4 @@ public class RequestedFeature public float lower; public float upper; } -} \ No newline at end of file +} diff --git a/Basis/Packages/HVRBasisAvatarComms/Scripts/Networking/StreamedAvatarFeature.cs b/Basis/Packages/HVRBasisAvatarComms/Scripts/Networking/StreamedAvatarFeature.cs index ae4449fad..639611c21 100644 --- a/Basis/Packages/HVRBasisAvatarComms/Scripts/Networking/StreamedAvatarFeature.cs +++ b/Basis/Packages/HVRBasisAvatarComms/Scripts/Networking/StreamedAvatarFeature.cs @@ -19,7 +19,8 @@ public class StreamedAvatarFeature : MonoBehaviour private const float EncodingRange = 254f; public DeliveryMethod DeliveryMethod = DeliveryMethod.Unreliable; - private const float TransmissionDeltaSeconds = 0.1f; + private const float TransmissionDeltaSecondsRegularSpeed = 0.1f; + private const float TransmissionDeltaSecondsRegularHighSpeed = 0.03f; internal BasisAvatar avatar; [SerializeField] public byte valueArraySize = 8; // Must not change after first enabled. @@ -34,10 +35,11 @@ public class StreamedAvatarFeature : MonoBehaviour private bool _writtenThisFrame; private bool _isWearer; private byte _scopedIndex; + private float _currentTransmissionDeltaSeconds = TransmissionDeltaSecondsRegularSpeed; public event InterpolatedDataChanged OnInterpolatedDataChanged; public delegate void InterpolatedDataChanged(float[] current); - + private void Awake() { previous ??= new float[valueArraySize]; @@ -71,7 +73,7 @@ private void OnSender() { _timeLeft += Time.deltaTime; - if (_timeLeft > TransmissionDeltaSeconds) + if (_timeLeft > _currentTransmissionDeltaSeconds) { var toSend = new StreamedAvatarFeaturePayload { @@ -80,11 +82,21 @@ private void OnSender() }; EncodeAndSubmit(toSend, null); - + _timeLeft = 0; } } + public void SwitchToHighSpeedTransmission() + { + _currentTransmissionDeltaSeconds = TransmissionDeltaSecondsRegularHighSpeed; + } + + public void SwitchToRegularSpeedTransmission() + { + _currentTransmissionDeltaSeconds = TransmissionDeltaSecondsRegularSpeed; + } + private void OnReceiver() { var timePassed = Time.deltaTime; @@ -93,14 +105,14 @@ private void OnReceiver() float totalQueueSeconds = 0; foreach (var payload in _queue) totalQueueSeconds += payload.DeltaTime; // Debug.Log($"Queue time is {totalQueueSeconds} seconds, size is {_queue.Count}"); - + while (_timeLeft <= 0 && _queue.TryDequeue(out var eval)) { // Debug.Log($"Unpacking delta {eval.DeltaTime} as {string.Join(',', eval.FloatValues.Select(f => $"{f}"))}"); var effectiveDeltaTime = _queue.Count <= 5 || totalQueueSeconds < 0.2f ? eval.DeltaTime : (eval.DeltaTime * Mathf.Lerp(0.66f, 0.05f, Mathf.InverseLerp(DeltaTimeUsedForResyncs, totalQueueSeconds, 4f))); - + _timeLeft += effectiveDeltaTime; previous = target; target = eval.FloatValues; @@ -153,7 +165,7 @@ public void SetEncodingInfo(bool isWearer, byte scopedIndex) public void OnPacketReceived(ArraySegment subBuffer) { if (!isActiveAndEnabled) return; - + if (TryDecode(subBuffer, out var result)) { _queue.Enqueue(result); @@ -171,12 +183,12 @@ private void EncodeAndSubmit(StreamedAvatarFeaturePayload message, ushort[] reci var buffer = new byte[HeaderBytes + valueArraySize]; buffer[0] = _scopedIndex; buffer[1] = (byte)(message.DeltaTime / DeltaLocalIntToSeconds); - + for (var i = 0; i < current.Length; i++) { buffer[HeaderBytes + i] = (byte)(message.FloatValues[i] * EncodingRange); } - + avatar.NetworkMessageSend(HVRAvatarComms.OurMessageIndex, buffer, DeliveryMethod, recipientsNullable); } @@ -192,16 +204,16 @@ private bool TryDecode(ArraySegment subBuffer, out StreamedAvatarFeaturePa { floatValues[i - SubHeaderBytes] = subBuffer.get_Item(i) / EncodingRange; } - + result = new StreamedAvatarFeaturePayload { DeltaTime = subBuffer.get_Item(0) * DeltaLocalIntToSeconds, FloatValues = floatValues }; - + return true; } - + #endregion public void OnResyncEveryoneRequested()