diff --git a/src/core/audio/hle_core.cpp b/src/core/audio/hle_core.cpp
index 23a99786..112db9d5 100644
--- a/src/core/audio/hle_core.cpp
+++ b/src/core/audio/hle_core.cpp
@@ -2,6 +2,7 @@
 
 #include <algorithm>
 #include <cassert>
+#include <iterator>
 #include <thread>
 #include <utility>
 
@@ -370,6 +371,13 @@ namespace Audio {
 				break;
 		}
 
+		// We're skipping the first samplePosition samples, so remove them from the buffer so as not to consume them later
+		if (source.samplePosition > 0) {
+			auto start = source.currentSamples.begin();
+			auto end = std::next(start, source.samplePosition);
+			source.currentSamples.erase(start, end);
+		}
+
 		// If the buffer is a looping buffer, re-push it
 		if (buffer.looping) {
 			source.pushBuffer(buffer);