Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: Feature/audio #11

Draft
wants to merge 8 commits into
base: develop
Choose a base branch
from
Draft
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add audio indicator using FBSlider
matinlotfali committed Oct 19, 2024
commit 1c32feacc5e2406b274b6db1b17d1ad2d58d26b3
2 changes: 1 addition & 1 deletion plugins/mobu/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@ project(MobuStreamDevice)

find_package(nng CONFIG REQUIRED)

set(SRCS plugin.cpp layout.cpp device.cpp device.h layout.h mobuModel.h mobuModel.cpp audio.cpp)
set(SRCS plugin.cpp layout.cpp device.cpp device.h layout.h mobuModel.h mobuModel.cpp audio.h audio.cpp)

function(build_mobu version path)
if(NOT EXISTS ${path})
84 changes: 76 additions & 8 deletions plugins/mobu/audio.cpp
Original file line number Diff line number Diff line change
@@ -8,6 +8,9 @@

#include <locale>
#include <system_error>
#include <thread>

constexpr int REFTIMES_PER_SEC = 10000000;

// Throw a std::system_error if the HRESULT indicates failure.
template<typename T>
@@ -17,7 +20,7 @@ void ThrowIfFailed(HRESULT hr, T&& msg) {
}
}

int WideCompare(const LPWSTR wstr, const char* str)
int WideCompare(LPWSTR wstr, const char* str)
{
wchar_t ws[255];
swprintf(ws, 255, L"%hs", str);
@@ -28,28 +31,91 @@ Open3D_AudioInput::Open3D_AudioInput() {
//auto hr = CoInitializeEx(nullptr, COINIT_MULTITHREADED);
//ThrowIfFailed(hr, "Failed to initialize the MMDeviceAPI.");

GUID IDevice_FriendlyName = {
mNameKey.pid = 14;
mNameKey.fmtid = {
0xa45c254e, 0xdf1c, 0x4efd,
{ 0x80, 0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0 }
};
mNameKey.pid = 14;
mNameKey.fmtid = IDevice_FriendlyName;
}

void Open3D_AudioInput::recorder_release() {
void Open3D_AudioInput::publish_audio_captured() const {
for (const auto subscriber : subscribers) {
subscriber->audio_captured(captureBuffer, nFrames);
}
}

void Open3D_AudioInput::activate() {
if (recorder == nullptr) {
return;
}

auto hr = recorder->Activate(__uuidof(IAudioClient), CLSCTX_ALL,
nullptr, reinterpret_cast<void**>(&recorderClient));
ThrowIfFailed(hr, "Failed to activate the recorder.");

hr = recorderClient->GetMixFormat(&format);
ThrowIfFailed(hr, "Failed to get the recording format.");

hr = recorderClient->Initialize(AUDCLNT_SHAREMODE_SHARED, 0, REFTIMES_PER_SEC,
0, format, nullptr);
ThrowIfFailed(hr, "Failed to initialize the recording client with the recording format.");

hr = recorderClient->GetService(__uuidof(IAudioCaptureClient), reinterpret_cast<void**>(&recorderService));
ThrowIfFailed(hr, "Failed to get the capture service for the recorder client.");

hr = recorderClient->Start();
ThrowIfFailed(hr, "Failed to start the recorder.");

recordThread = new std::thread(&Open3D_AudioInput::loop, this);
}

void Open3D_AudioInput::release() {
if (recordThread) {
stopThread = true;
recordThread->join();
delete recordThread;
recordThread = nullptr;
stopThread = false;
}
if (recorderClient) {
recorderClient->Stop();

if (recorderService) {
recorderService->Release();
recorderService = nullptr;
}
recorderClient->Release();
recorderClient = nullptr;
}
if (recorder) {
recorder->Release();
recorder = nullptr;
}
}

void Open3D_AudioInput::loop() {
HRESULT hr;
while (!stopThread) {
hr = recorderService->GetBuffer(&captureBuffer, &nFrames, &flags, nullptr, nullptr);
ThrowIfFailed(hr, "Failed to get recording buffer.");

if (captureBuffer == nullptr)
continue;

hr = recorderService->ReleaseBuffer(nFrames);
ThrowIfFailed(hr, "Failed to release recording buffer");

publish_audio_captured();
}
}

Open3D_AudioInput::~Open3D_AudioInput() {
recorder_release();
release();
//CoUninitialize();
}

void Open3D_AudioInput::set_device(FBAudioIn *mic) {
recorder_release();
release();
if (mic == nullptr) {
return;
}
@@ -86,7 +152,7 @@ void Open3D_AudioInput::set_device(FBAudioIn *mic) {

if (varName.vt != VT_EMPTY) {
if (WideCompare(varName.pwszVal, mic->Name.AsString()) == 0) {
recorder = device;
recorder = device;
properties->Release();
break;
}
@@ -97,4 +163,6 @@ void Open3D_AudioInput::set_device(FBAudioIn *mic) {
}
recorderCollection->Release();
enumerator->Release();

activate();
}
41 changes: 38 additions & 3 deletions plugins/mobu/audio.h
Original file line number Diff line number Diff line change
@@ -7,26 +7,61 @@

#include <Audioclient.h>
#include <fbsdk/fbsdk.h>
#include <vector>

struct IMMDevice;
struct IAudioClient;
struct IAudioRenderClient;
namespace std {
class thread;
}

class IAudioSubscriber {
public:
virtual ~IAudioSubscriber() = default;
virtual void audio_captured(const BYTE* captureBuffer, UINT32 nFrames) = 0;
};

class Open3D_AudioInput
{
public:
Open3D_AudioInput();
~Open3D_AudioInput();
void set_device(FBAudioIn* mic);
void subscribe(IAudioSubscriber* subscriber) { subscribers.push_back(subscriber); }
void unsubscribe(IAudioSubscriber* subscriber) { subscribers.erase(std::remove(subscribers.begin(), subscribers.end(), subscriber), subscribers.end()); }

private:
// Audio Device Activation Section
// ===============================

IMMDevice* recorder = nullptr;
IAudioClient* recorderClient = nullptr;
IAudioRenderClient* renderService = nullptr;
IAudioCaptureClient* recorderService = nullptr;

WAVEFORMATEX* format = nullptr;
PROPERTYKEY mNameKey;
PROPERTYKEY mNameKey = {};

void activate();
void release();

// Audio Capture Loop Section
// ==========================

BYTE* captureBuffer = nullptr;
UINT32 nFrames = 0;
DWORD flags = 0;

std::thread* recordThread = nullptr;
bool stopThread = false;

void loop();

// Audio Capture Publishing to Subscribers Section
// ===============================================

void recorder_release();
std::vector<IAudioSubscriber*> subscribers;
void publish_audio_captured() const;
};


6 changes: 3 additions & 3 deletions plugins/mobu/device.h
Original file line number Diff line number Diff line change
@@ -83,6 +83,8 @@ class Open3D_Device : public FBDevice
SOCKET mNetworkSocket;
std::vector<int> mClients;

Open3D_AudioInput mAudioRecord;

private:
double mSamplingRate;
FBDeviceSamplingMode mSamplingType;
@@ -94,9 +96,7 @@ class Open3D_Device : public FBDevice
int mNetworkPort;
bool mStreaming;
int mFrameCounter;
uint32_t mIdSeq;

Open3D_AudioInput mAudioRecord;
uint32_t mIdSeq;
};

#endif
35 changes: 32 additions & 3 deletions plugins/mobu/layout.cpp
Original file line number Diff line number Diff line change
@@ -20,6 +20,7 @@ bool Open3D_Device_Layout::FBCreate()
// Add device & system callbacks
mDevice->OnStatusChange.Add ( this,(FBCallback)&Open3D_Device_Layout::EventDeviceStatusChange );
OnIdle.Add ( this,(FBCallback)&Open3D_Device_Layout::EventUIIdle );
mDevice->mAudioRecord.subscribe(this);

return true;
}
@@ -29,6 +30,7 @@ void Open3D_Device_Layout::FBDestroy()
// Remove device & system callbacks
OnIdle.Remove ( this,(FBCallback)&Open3D_Device_Layout::EventUIIdle );
mDevice->OnStatusChange.Remove ( this,(FBCallback)&Open3D_Device_Layout::EventDeviceStatusChange );
mDevice->mAudioRecord.unsubscribe(this);
}

void Open3D_Device_Layout::UICreate()
@@ -213,7 +215,24 @@ void Open3D_Device_Layout::UICreate()
lH, kFBAttachNone, NULL, 1.00);
mLayoutLeft.SetControl("ListMicSource", mListMicSource);
mListMicSource.OnChange.Add(this, (FBCallback)&Open3D_Device_Layout::EventSelectMicSource);
mListMicSource.Style = FBListStyle::kFBDropDownList;
mListMicSource.Style = kFBDropDownList;

// Microphone Capture Indicator - Under Microphones List
mLayoutLeft.AddRegion("MicCaptureIndicator", "MicCaptureIndicator",
lS, kFBAttachRight, "SourcesList", 1.00,
5, kFBAttachBottom, "ListMicSource", 1.00,
155, kFBAttachNone, "", 1.00,
lH, kFBAttachNone, NULL, 1.00);
mLayoutLeft.SetControl("MicCaptureIndicator", mMicCaptureIndicator);
mMicCaptureIndicator.Height = 10;
mMicCaptureIndicator.Max = 0;
mMicCaptureIndicator.Max = 1;
mMicCaptureIndicator.Value = 0;
mMicCaptureIndicator.LargeStep = 0.01;
mMicCaptureIndicator.SmallStep = 0.01;
mMicCaptureIndicator.Visible = false;
mMicCaptureIndicator.Orientation = kFBHorizontal;
mMicCaptureIndicator.ReadOnly = true;

// Version info - at bottom
mLayoutLeft.AddRegion("VersionInfo", "VersionInfo",
@@ -416,6 +435,12 @@ void Open3D_Device_Layout::EventEditDelta(HISender pSender, HKEvent pEvent)
mDevice->SetDeltaThreshold(value);
}

void Open3D_Device_Layout::audio_captured(const BYTE* captureBuffer, const UINT32 nFrames)
{
const auto captureBufferFloat = reinterpret_cast<const float*>(captureBuffer);
mMicCaptureIndicator.Value = fabsf(captureBufferFloat[0]);
}

void Open3D_Device_Layout::EventDeviceStatusChange( HISender pSender, HKEvent pEvent )
{
UIReset();
@@ -613,10 +638,14 @@ void Open3D_Device_Layout::EventEditJoints(HISender pSender, HKEvent pEvent)
}

void Open3D_Device_Layout::EventSelectMicSource(HISender pSender, HKEvent pEvent) {
if (mListMicSource.ItemIndex == 0)
if (mListMicSource.ItemIndex == 0) {
mDevice->SetSelectedMicrophone(nullptr);
mMicCaptureIndicator.Value = 0;
mMicCaptureIndicator.Visible = false;
}
else {
auto& micList = FBSystem::TheOne().AudioInputs;
mDevice->SetSelectedMicrophone(micList[mListMicSource.ItemIndex - 1]);
mMicCaptureIndicator.Visible = true;
}
}
}
4 changes: 3 additions & 1 deletion plugins/mobu/layout.h
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@

#include "device.h"

class Open3D_Device_Layout : public FBDeviceLayout
class Open3D_Device_Layout : public FBDeviceLayout, IAudioSubscriber
{
FBDeviceLayoutDeclare( Open3D_Device_Layout, FBDeviceLayout );
public:
@@ -34,6 +34,7 @@ class Open3D_Device_Layout : public FBDeviceLayout
void EventEditSampleRate(HISender pSender, HKEvent pEvent);
void EventEditKey(HISender pSender, HKEvent pEvent);
void EventEditDelta(HISender pSender, HKEvent pEvent);
void audio_captured(const BYTE* captureBuffer, UINT32 nFrames) override;

private:
FBLayout mLayoutLeft;
@@ -61,6 +62,7 @@ class Open3D_Device_Layout : public FBDeviceLayout
FBEditNumber mEditSamplingRate;
FBLabel mLabelMicSource;
FBList mListMicSource;
FBSlider mMicCaptureIndicator;
FBLabel mLabelPluginVersion;
FBLabel mLabelJoints;
FBEdit mMemoJoints;