Get audio output working with LLE DSP (#419)

* Implement audio output

* Semi-proper audio output

* Add audio enable and vsync settings

* Add audio enable and vsync settings

* Optimize audio output a bit

* Make max ring buffer timeout smaller

* Make max ring buffer timeout smaller

* Revert to spinlocking for audio sync

* Sleep emulator thread if too many samples queued

* Fix Teakra submodule breaking

* Don't start audio device too soon

* Fix IWYU errors

* Fix compilation errors on GCC/Clang

* Ignore std::hardware_destructive_interference_size on Android NDK

* Fix more IWYU errors

src/config.cpp

@@ -59,6 +59,7 @@ void EmulatorConfig::load() {
 			}
 
 			shaderJitEnabled = toml::find_or<toml::boolean>(gpu, "EnableShaderJIT", shaderJitDefault);
+			vsyncEnabled = toml::find_or<toml::boolean>(gpu, "EnableVSync", true);
 		}
 	}
 
@@ -69,6 +70,7 @@ void EmulatorConfig::load() {
 
 			auto dspCoreName = toml::find_or<std::string>(audio, "DSPEmulation", "Null");
 			dspType = Audio::DSPCore::typeFromString(dspCoreName);
+			audioEnabled = toml::find_or<toml::boolean>(audio, "EnableAudio", false);
 		}
 	}
 
@@ -119,7 +121,9 @@ void EmulatorConfig::save() {
 	data["General"]["UsePortableBuild"] = usePortableBuild;
 	data["GPU"]["EnableShaderJIT"] = shaderJitEnabled;
 	data["GPU"]["Renderer"] = std::string(Renderer::typeToString(rendererType));
+	data["GPU"]["EnableVSync"] = vsyncEnabled;
 	data["Audio"]["DSPEmulation"] = std::string(Audio::DSPCore::typeToString(dspType));
+	data["Audio"]["EnableAudio"] = audioEnabled;
 
 	data["Battery"]["ChargerPlugged"] = chargerPlugged;
 	data["Battery"]["BatteryPercentage"] = batteryPercentage;

src/core/audio/miniaudio_device.cpp

@@ -0,0 +1,143 @@
+#include "audio/miniaudio_device.hpp"
+
+#include "helpers.hpp"
+
+MiniAudioDevice::MiniAudioDevice() : initialized(false), running(false), samples(nullptr) {}
+
+void MiniAudioDevice::init(Samples& samples, bool safe) {
+	this->samples = &samples;
+	running = false;
+
+	// Probe for device and available backends and initialize audio
+	ma_backend backends[ma_backend_null + 1];
+	uint count = 0;
+
+	if (safe) {
+		backends[0] = ma_backend_null;
+		count = 1;
+	} else {
+		bool found = false;
+		for (uint i = 0; i <= ma_backend_null; i++) {
+			ma_backend backend = ma_backend(i);
+			if (!ma_is_backend_enabled(backend)) {
+				continue;
+			}
+
+			backends[count++] = backend;
+
+			// TODO: Make backend selectable here
+			found = true;
+			//count = 1;
+			//backends[0] = backend;
+		}
+
+		if (!found) {
+			initialized = false;
+
+			Helpers::warn("No valid audio backend found\n");
+			return;
+		}
+	}
+
+	if (ma_context_init(backends, count, nullptr, &context) != MA_SUCCESS) {
+		initialized = false;
+
+		Helpers::warn("Unable to initialize audio context");
+		return;
+	}
+	audioDevices.clear();
+
+	struct UserContext {
+		MiniAudioDevice* miniAudio;
+		ma_device_config& config;
+		bool found = false;
+	};
+	UserContext userContext = {.miniAudio = this, .config = deviceConfig};
+
+	ma_context_enumerate_devices(
+		&context,
+		[](ma_context* pContext, ma_device_type deviceType, const ma_device_info* pInfo, void* pUserData) -> ma_bool32 {
+			if (deviceType != ma_device_type_playback) {
+				return true;
+			}
+
+			UserContext* userContext = reinterpret_cast<UserContext*>(pUserData);
+			userContext->miniAudio->audioDevices.push_back(pInfo->name);
+
+			// TODO: Check if this is the device we want here
+			userContext->config.playback.pDeviceID = &pInfo->id;
+			userContext->found = true;
+			return true;
+		},
+		&userContext
+	);
+
+	if (!userContext.found) {
+		Helpers::warn("MiniAudio: Device not found");
+	}
+
+	deviceConfig = ma_device_config_init(ma_device_type_playback);
+	// The 3DS outputs s16 stereo audio @ 32768 Hz
+	deviceConfig.playback.format = ma_format_s16;
+	deviceConfig.playback.channels = channelCount;
+	deviceConfig.sampleRate = sampleRate;
+	//deviceConfig.periodSizeInFrames = 64;
+	//deviceConfig.periods = 16;
+	deviceConfig.pUserData = this;
+	deviceConfig.aaudio.usage = ma_aaudio_usage_game;
+	deviceConfig.wasapi.noAutoConvertSRC = true;
+
+	deviceConfig.dataCallback = [](ma_device* device, void* out, const void* input, ma_uint32 frameCount) {
+		auto self = reinterpret_cast<MiniAudioDevice*>(device->pUserData);
+		s16* output = reinterpret_cast<ma_int16*>(out);
+
+		// Wait until there's enough samples to pop
+		while (self->samples->size() < frameCount * channelCount) {
+			// If audio output is disabled from the emulator thread, make sure that this callback will return and not hang
+			if (!self->running) {
+				return;
+			}
+		}
+
+		self->samples->pop(output, frameCount * channelCount);
+	};
+
+	if (ma_device_init(&context, &deviceConfig, &device) != MA_SUCCESS) {
+		Helpers::warn("Unable to initialize audio device");
+		initialized = false;
+		return;
+	}
+
+	initialized = true;
+}
+
+void MiniAudioDevice::start() {
+	if (!initialized) {
+		Helpers::warn("MiniAudio device not initialized, won't start");
+		return;
+	}
+
+	// Ignore the call to start if the device is already running
+	if (!running) {
+		if (ma_device_start(&device) == MA_SUCCESS) {
+			running = true;
+		} else {
+			Helpers::warn("Failed to start audio device");
+		}
+	}
+}
+
+void MiniAudioDevice::stop() {
+	if (!initialized) {
+		Helpers::warn("MiniAudio device not initialized, can't start");
+		return;
+	}
+
+	if (running) {
+		running = false;
+
+		if (ma_device_stop(&device) != MA_SUCCESS) {
+			Helpers::warn("Failed to stop audio device");
+		} 
+	}
+}

src/core/audio/teakra_core.cpp

@@ -1,7 +1,9 @@
 #include "audio/teakra_core.hpp"
 
 #include <algorithm>
+#include <chrono>
 #include <cstring>
+#include <thread>
 
 #include "services/dsp.hpp"
 
@@ -51,10 +53,7 @@ TeakraDSP::TeakraDSP(Memory& mem, Scheduler& scheduler, DSPService& dspService)
 	ahbm.write32 = [&](u32 addr, u32 value) { *(u32*)&mem.getFCRAM()[addr - PhysicalAddrs::FCRAM] = value; };
 
 	teakra.SetAHBMCallback(ahbm);
-	teakra.SetAudioCallback([=](std::array<s16, 2> sample) {
-		//printf("%d %d\n", sample[0], sample[1]);
-		// NOP for now
-	});
+	teakra.SetAudioCallback([](std::array<s16, 2> sample) { /* Do nothing */ });
 
 	// Set up event handlers. These handlers forward a hardware interrupt to the DSP service, which is responsible
 	// For triggering the appropriate DSP kernel events
@@ -114,6 +113,36 @@ void TeakraDSP::reset() {
 	running = false;
 	loaded = false;
 	signalledData = signalledSemaphore = false;
+
+	audioFrameIndex = 0;
+}
+
+void TeakraDSP::setAudioEnabled(bool enable) {
+	if (audioEnabled != enable) {
+		audioEnabled = enable;
+
+		// Set the appropriate audio callback for Teakra
+		if (audioEnabled) {
+			teakra.SetAudioCallback([this](std::array<s16, 2> sample) {
+				audioFrame[audioFrameIndex++] = sample[0];
+				audioFrame[audioFrameIndex++] = sample[1];
+
+				// Push our samples at the end of an audio frame
+				if (audioFrameIndex >= audioFrame.size()) {
+					audioFrameIndex -= audioFrame.size();
+
+					// Wait until we've actually got room to do so
+					while (sampleBuffer.size() + 2 > sampleBuffer.Capacity()) {
+						std::this_thread::sleep_for(std::chrono::milliseconds{1});
+					}
+
+					sampleBuffer.push(audioFrame.data(), audioFrame.size());
+				}
+			});
+		} else {
+			teakra.SetAudioCallback([](std::array<s16, 2> sample) { /* Do nothing */ });
+		}
+	}
 }
 
 // https://github.com/citra-emu/citra/blob/master/src/audio_core/lle/lle.cpp
@@ -313,4 +342,4 @@ void TeakraDSP::unloadComponent() {
 	// Read the value and discard it, completing shutdown
 	teakra.RecvData(2);
 	running = false;
-}
+}

src/emulator.cpp

@@ -29,6 +29,9 @@ Emulator::Emulator()
 	dsp = Audio::makeDSPCore(config.dspType, memory, scheduler, dspService);
 	dspService.setDSPCore(dsp.get());
 
+	audioDevice.init(dsp->getSamples());
+	setAudioEnabled(config.audioEnabled);
+
 #ifdef PANDA3DS_ENABLE_DISCORD_RPC
 	if (config.discordRpcEnabled) {
 		discordRpc.init();
@@ -102,8 +105,19 @@ void Emulator::step() {}
 void Emulator::render() {}
 
 // Only resume if a ROM is properly loaded
-void Emulator::resume() { running = (romType != ROMType::None); }
-void Emulator::pause() { running = false; }
+void Emulator::resume() {
+	running = (romType != ROMType::None);
+
+	if (running) {
+		audioDevice.start();
+	}
+}
+
+void Emulator::pause() {
+	running = false;
+	audioDevice.stop();
+}
+
 void Emulator::togglePause() { running ? pause() : resume(); }
 
 void Emulator::runFrame() {
@@ -387,4 +401,16 @@ RomFS::DumpingResult Emulator::dumpRomFS(const std::filesystem::path& path) {
 	dumpRomFSNode(*node, (const char*)&romFS[0], path);
 
 	return DumpingResult::Success;
+}
+
+void Emulator::setAudioEnabled(bool enable) {
+	if (!enable) {
+		audioDevice.stop();
+	} else if (enable && romType != ROMType::None && running) {
+		// Don't start the audio device yet if there's no ROM loaded or the emulator is paused
+		// Resume and Pause will handle it
+		audioDevice.start();
+	}
+
+	dsp->setAudioEnabled(enable);
 }

src/miniaudio.cpp

@@ -0,0 +1,7 @@
+// We do not need the ability to be able to encode or decode audio files for the time being
+// So we disable said functionality to make the executable smaller
+#define MA_NO_DECODING
+#define MA_NO_ENCODING
+#define MINIAUDIO_IMPLEMENTATION
+
+#include "miniaudio.h"

src/panda_qt/main_window.cpp

@@ -87,7 +87,7 @@ MainWindow::MainWindow(QApplication* app, QWidget* parent) : QMainWindow(parent)
 			// Make GL context current for this thread, enable VSync
 			GL::Context* glContext = screen.getGLContext();
 			glContext->MakeCurrent();
-			glContext->SetSwapInterval(1);
+			glContext->SetSwapInterval(emu->getConfig().vsyncEnabled ? 1 : 0);
 
 			emu->initGraphicsContext(glContext);
 		} else if (usingVk) {

src/panda_sdl/frontend_sdl.cpp

@@ -49,6 +49,8 @@ FrontendSDL::FrontendSDL() {
 		if (!gladLoadGLLoader(reinterpret_cast<GLADloadproc>(SDL_GL_GetProcAddress))) {
 			Helpers::panic("OpenGL init failed");
 		}
+
+		SDL_GL_SetSwapInterval(config.vsyncEnabled ? 1 : 0);
 	}
 
 #ifdef PANDA3DS_ENABLE_VULKAN