Merge pull request #110 from jdibenes/realsense-435i-test

Extended audio/video interfaces

README.md

@@ -26,7 +26,8 @@ HoloLens 2 server software and Python client library for streaming sensor data v
 - Scene Understanding (3D Meshes + Semantic labels for planar surfaces)
 - Voice Input
 - Extended Eye Tracking (30, 60, or 90 FPS)
-- Extended Audio (Microphone + Application audio, 2 channels @ 48000 Hz, 16-bit PCM, AAC encoded)
+- Extended Audio (Internal Microphone or External USB-C Microphone + Application audio, 2 channels @ 48000 Hz, 16-bit PCM, AAC encoded)
+- Extended Video (Front Camera or External USB-C camera)
 
 **Additional features**
 

etc/realsense435i_select.txt

@@ -0,0 +1,9 @@
+PV
+g_index = 1
+s_index = 2
+p_index = 4
+
+RealSense RGB
+g_index = 2
+s_index = 0
+p_index = 0

hl2ss/hl2ss/Package.appxmanifest

@@ -9,7 +9,7 @@
   <Identity
     Name="eaaf3af3-1402-4e5b-b6a1-5d0fbb7c1ba8"
     Publisher="CN=jcds"
-    Version="1.0.28.0" />
+    Version="1.0.29.0" />
   <mp:PhoneIdentity PhoneProductId="eaaf3af3-1402-4e5b-b6a1-5d0fbb7c1ba8" PhonePublisherId="00000000-0000-0000-0000-000000000000"/>
   <Properties>
     <DisplayName>hl2ss</DisplayName>

hl2ss/hl2ss/custom_audio_effect.cpp

@@ -6,7 +6,7 @@ using namespace winrt::Windows::Media::Effects;
 
 MRCAudioEffect::MRCAudioEffect(MRCAudioOptions const& options)
 {
-    m_propertySet.Insert(L"MixerMode", winrt::box_value<uint32_t>(options.mixer_mode));
+    m_propertySet.Insert(L"MixerMode", winrt::box_value<uint32_t>((options.mixer_mode & 3) % 3));
     m_propertySet.Insert(L"LoopbackGain", winrt::box_value<float>(options.loopback_gain));
     m_propertySet.Insert(L"MicrophoneGain", winrt::box_value<float>(options.microphone_gain));
 }

hl2ss/hl2ss/custom_media_types.cpp

@@ -196,6 +196,78 @@ static HRESULT CreateTypeARGB(IMFMediaType** ppType, uint32_t width, uint32_t he
     return S_OK;
 }
 
+// https://learn.microsoft.com/en-us/windows/win32/medfound/video-subtype-guids
+// OK
+static HRESULT CreateTypeYUY2(IMFMediaType** ppType, uint32_t width, uint32_t height, uint32_t stride, uint32_t fps_num, uint32_t fps_den)
+{
+    IMFMediaType* pType;
+
+    MFCreateMediaType(&pType);
+
+    pType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+    pType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_YUY2);
+    pType->SetUINT32(MF_MT_DEFAULT_STRIDE, stride);
+    MFSetAttributeRatio(pType, MF_MT_FRAME_RATE, fps_num, fps_den);
+    MFSetAttributeSize(pType, MF_MT_FRAME_SIZE, width, height);
+    pType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlaceMode::MFVideoInterlace_Progressive);
+    pType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE);
+    MFSetAttributeRatio(pType, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
+    pType->SetUINT32(MF_MT_SAMPLE_SIZE, 2UL * width * height);
+    pType->SetUINT32(MF_MT_FIXED_SIZE_SAMPLES, TRUE);
+
+    *ppType = pType;
+
+    return S_OK;
+}
+
+// https://learn.microsoft.com/en-us/windows/win32/medfound/video-subtype-guids
+// OK
+static HRESULT CreateTypeIYUV(IMFMediaType** ppType, uint32_t width, uint32_t height, uint32_t stride, uint32_t fps_num, uint32_t fps_den)
+{
+    IMFMediaType* pType;
+
+    MFCreateMediaType(&pType);
+
+    pType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+    pType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_IYUV);
+    pType->SetUINT32(MF_MT_DEFAULT_STRIDE, stride);
+    MFSetAttributeRatio(pType, MF_MT_FRAME_RATE, fps_num, fps_den);
+    MFSetAttributeSize(pType, MF_MT_FRAME_SIZE, width, height);
+    pType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlaceMode::MFVideoInterlace_Progressive);
+    pType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE);
+    MFSetAttributeRatio(pType, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
+    pType->SetUINT32(MF_MT_SAMPLE_SIZE, (3UL * width * height) / 2UL);
+    pType->SetUINT32(MF_MT_FIXED_SIZE_SAMPLES, TRUE);
+
+    *ppType = pType;
+
+    return S_OK;
+}
+
+// https://learn.microsoft.com/en-us/windows/win32/medfound/video-subtype-guids
+// OK
+static HRESULT CreateTypeYV12(IMFMediaType** ppType, uint32_t width, uint32_t height, uint32_t stride, uint32_t fps_num, uint32_t fps_den)
+{
+    IMFMediaType* pType;
+
+    MFCreateMediaType(&pType);
+
+    pType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
+    pType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_YV12);
+    pType->SetUINT32(MF_MT_DEFAULT_STRIDE, stride);
+    MFSetAttributeRatio(pType, MF_MT_FRAME_RATE, fps_num, fps_den);
+    MFSetAttributeSize(pType, MF_MT_FRAME_SIZE, width, height);
+    pType->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlaceMode::MFVideoInterlace_Progressive);
+    pType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE);
+    MFSetAttributeRatio(pType, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
+    pType->SetUINT32(MF_MT_SAMPLE_SIZE, (3UL * width * height) / 2UL);
+    pType->SetUINT32(MF_MT_FIXED_SIZE_SAMPLES, TRUE);
+
+    *ppType = pType;
+
+    return S_OK;
+}
+
 // https://docs.microsoft.com/en-us/windows/win32/medfound/h-264-video-encoder
 // OK
 static HRESULT CreateTypeH264(IMFMediaType** ppType, uint32_t width, uint32_t height, uint32_t fps_num, uint32_t fps_den, eAVEncH264VProfile profile, int32_t level, uint32_t bitrate)
@@ -255,6 +327,9 @@ HRESULT CreateTypeVideo(IMFMediaType** ppType, uint32_t width, uint32_t height,
     if (subtype == VideoSubtype::VideoSubtype_L16)  { return CreateTypeL16( ppType, width, height, stride, fps_num, fps_den); }
     if (subtype == VideoSubtype::VideoSubtype_NV12) { return CreateTypeNV12(ppType, width, height, stride, fps_num, fps_den); }
     if (subtype == VideoSubtype::VideoSubtype_ARGB) { return CreateTypeARGB(ppType, width, height, stride, fps_num, fps_den); }
+    if (subtype == VideoSubtype::VideoSubtype_YUY2) { return CreateTypeYUY2(ppType, width, height, stride, fps_num, fps_den); }
+    if (subtype == VideoSubtype::VideoSubtype_IYUV) { return CreateTypeIYUV(ppType, width, height, stride, fps_num, fps_den); }
+    if (subtype == VideoSubtype::VideoSubtype_YV12) { return CreateTypeYV12(ppType, width, height, stride, fps_num, fps_den); }
 
     *ppType = NULL;
 

hl2ss/hl2ss/custom_media_types.h

@@ -50,7 +50,10 @@ enum VideoSubtype : uint8_t
     VideoSubtype_L8,
     VideoSubtype_L16,
     VideoSubtype_NV12,
-    VideoSubtype_ARGB
+    VideoSubtype_ARGB,
+    VideoSubtype_YUY2,
+    VideoSubtype_IYUV,
+    VideoSubtype_YV12
 };
 
 HRESULT CreateTypeAudio(IMFMediaType** ppType, uint32_t channels, uint32_t samplerate, AudioSubtype subtype, AACProfile profile, AACLevel level);

hl2ss/hl2ss/extended_audio.cpp

@@ -1,21 +1,38 @@
 
+#include <mfapi.h>
+#include <combaseapi.h>
+#include "custom_media_types.h"
 #include "custom_audio_effect.h"
+#include "nfo.h"
+#include "log.h"
 
 #include <winrt/Windows.Foundation.h>
 #include <winrt/Windows.Foundation.Collections.h>
 #include <winrt/Windows.Media.MediaProperties.h>
 #include <winrt/Windows.Media.Devices.h>
+#include <winrt/Windows.Devices.Enumeration.h>
 #include <winrt/Windows.Media.Capture.h>
 #include <winrt/Windows.Media.Capture.Frames.h>
+#include <winrt/Windows.Data.Json.h>
 
+using namespace winrt::Windows::Media::MediaProperties;
 using namespace winrt::Windows::Media::Devices;
 using namespace winrt::Windows::Media::Capture;
 using namespace winrt::Windows::Media::Capture::Frames;
+using namespace winrt::Windows::Data::Json;
+
+struct source_format
+{
+    MediaFrameSource source;
+    MediaFrameFormat format;
+};
 
 //-----------------------------------------------------------------------------
 // Global Variables
 //-----------------------------------------------------------------------------
 
+static HANDLE g_event = NULL;
+
 static MediaCapture g_mediaCapture = nullptr;
 static MediaFrameSource g_audioSource = nullptr;
 
@@ -24,26 +41,124 @@ static MediaFrameSource g_audioSource = nullptr;
 //-----------------------------------------------------------------------------
 
 // OK
-void ExtendedAudio_Open(MRCAudioOptions const& options)
+static void ExtendedAudio_OnFailed(MediaCapture const&, MediaCaptureFailedEventArgs const& b)
+{
+    ShowMessage(L"ExtendedAudio_OnFailed - 0x%X : '%s'", b.Code(), b.Message().c_str());
+    if (g_event != NULL) { SetEvent(g_event); }
+}
+
+// OK
+static bool ExtendedAudio_ParseSubtype(winrt::hstring const& s, AudioSubtype& v)
 {
+    if (s == L"PCM")   { v = AudioSubtype::AudioSubtype_S16; return true; }
+    if (s == L"Float") { v = AudioSubtype::AudioSubtype_F32; return true; }
+    return false;
+}
+
+// OK
+static bool ExtendedAudio_FindAudioSource(MediaCapture const& mediaCapture, MediaFrameSource& audioSource)
+{
+    std::vector<source_format> sources[2][2];
+
+    for (auto const& frameSource : mediaCapture.FrameSources())
+    {
+    auto source = frameSource.Value();
+    if (source.Info().MediaStreamType() != MediaStreamType::Audio) { continue; }
+    for (auto const& format : source.SupportedFormats())
+    {
+    auto aep = format.AudioEncodingProperties();
+    if (aep.SampleRate() != 48000) { continue; }
+    AudioSubtype subtype;
+    bool ok = ExtendedAudio_ParseSubtype(aep.Subtype(), subtype);
+    if (!ok) { continue; }
+    uint32_t channels = aep.ChannelCount();
+    if ((channels != 1) && (channels != 2)) { continue; }
+    sources[subtype][2 - channels].push_back({ source, format });
+    }
+    }
+
+    for (int i = 0; i < 2; ++i)
+    {
+    for (int j = 0; j < 2; ++j)
+    {
+    if (sources[i][j].size() <= 0) { continue; }
+    auto const& sf = sources[i][j][0];
+    audioSource = sf.source;
+    audioSource.SetFormatAsync(sf.format).get();
+    return true;
+    }
+    }
+
+    return false;
+}
+
+// OK
+void ExtendedAudio_QueryDevices(winrt::hstring& out)
+{
+    std::vector<winrt::hstring> ids;
+    std::vector<winrt::hstring> names;
+
+    GetAudioCaptureIdsAndNames(ids, names);
+
+    JsonObject root = JsonObject();
+    for (uint32_t i = 0; i < ids.size(); ++i)
+    {
+    JsonObject jsourceinfos = JsonObject();
+    jsourceinfos.Insert(L"Id", JsonValue::CreateStringValue(ids[i]));
+    jsourceinfos.Insert(L"Name", JsonValue::CreateStringValue(names[i]));
+
+    root.Insert(winrt::to_hstring(i), jsourceinfos);
+    }
+
+    out = root.ToString();
+}
+
+// OK
+void ExtendedAudio_RegisterEvent(HANDLE h)
+{
+    g_event = h;
+}
+
+// OK
+bool ExtendedAudio_Open(MRCAudioOptions const& options)
+{
+    uint32_t index = (options.mixer_mode & 0x7FFFFFFC) >> 2;
     MediaCaptureInitializationSettings settings;
+    std::vector<winrt::hstring> ids;
+    winrt::hstring id;
+    bool ok;
 
-    g_mediaCapture = MediaCapture();
+    if (index <= 0)
+    {
+    id = MediaDevice::GetDefaultAudioCaptureId(AudioDeviceRole::Default);
+    }
+    else
+    {
+    index--;
+    GetAudioCaptureIds(ids);
+    if (index >= ids.size()) { return false; }
+    id = ids[index];
+    }
 
-    settings.AudioDeviceId(MediaDevice::GetDefaultAudioCaptureId(AudioDeviceRole::Default));
+    settings.AudioDeviceId(id);
     settings.StreamingCaptureMode(StreamingCaptureMode::Audio);
     settings.SharingMode(MediaCaptureSharingMode::SharedReadOnly);
     settings.MediaCategory(MediaCategory::Media);
 
+    g_mediaCapture = MediaCapture();
     g_mediaCapture.InitializeAsync(settings).get();
     g_mediaCapture.AddAudioEffectAsync(MRCAudioEffect(options)).get();
 
-    for (auto const& frameSource : g_mediaCapture.FrameSources())
+    g_mediaCapture.Failed({ ExtendedAudio_OnFailed });
+    ok = ExtendedAudio_FindAudioSource(g_mediaCapture, g_audioSource);
+    if (!ok)
     {
-    auto const& frameSourceInfo = frameSource.Value().Info();
-    if (frameSourceInfo.MediaStreamType() != MediaStreamType::Audio) { continue; }
-    g_audioSource = frameSource.Value();
+    g_mediaCapture.Close();
+    g_mediaCapture = nullptr;
+    return false;
     }
+
+    return true;
 }
 
 // OK

hl2ss/hl2ss/extended_audio.h

@@ -1,9 +1,12 @@
 
 #pragma once
 
+#include "custom_media_types.h"
 #include "custom_audio_effect.h"
 #include <winrt/Windows.Media.Capture.Frames.h>
 
-void ExtendedAudio_Open(MRCAudioOptions const& options);
+void ExtendedAudio_QueryDevices(winrt::hstring& out);
+void ExtendedAudio_RegisterEvent(HANDLE h);
+bool ExtendedAudio_Open(MRCAudioOptions const& options);
 void ExtendedAudio_Close();
 winrt::Windows::Media::Capture::Frames::MediaFrameReader ExtendedAudio_CreateFrameReader();