diff --git a/include/audio/hle_core.hpp b/include/audio/hle_core.hpp
index b3832d76..eed7bcc1 100644
--- a/include/audio/hle_core.hpp
+++ b/include/audio/hle_core.hpp
@@ -61,6 +61,10 @@ namespace Audio {
 		// There's one gain configuration for each of the 3 intermediate mixing stages
 		// And each gain configuration is composed of 4 gain values, one for each sample in a quad-channel sample
 		std::array<std::array<float, 4>, 3> gains;
+		// Of the 3 intermediate mix stages, typically only the first one is actually enabled and the other ones do nothing
+		// Ie their gain is vec4(0.0). We track which stages are disabled (have a gain of all 0s) using this bitfield and skip them
+		// In order to save up on CPU time.
+		uint enabledMixStages = 0;
 
 		u32 samplePosition;  // Sample number into the current audio buffer
 		float rateMultiplier;
diff --git a/src/core/audio/hle_core.cpp b/src/core/audio/hle_core.cpp
index 77ed719f..be754c81 100644
--- a/src/core/audio/hle_core.cpp
+++ b/src/core/audio/hle_core.cpp
@@ -255,6 +255,11 @@ namespace Audio {
 			// If the source is still enabled, mix its output into the intermediate mix buffers
 			if (source.enabled) {
 				for (int mix = 0; mix < mixes.size(); mix++) {
+					// Check if this stage is passthrough, and if it is, then skip it
+					if ((source.enabledMixStages & (1u << mix)) == 0) {
+						continue;
+					}
+
 					IntermediateMix& intermediateMix = mixes[mix];
 					const std::array<float, 4>& gains = source.gains[mix];
 
@@ -397,16 +402,23 @@ namespace Audio {
 			}
 		}
 
-#define CONFIG_GAIN(index)                 \
-	if (config.gain##index##Dirty) {       \
-		auto& dest = source.gains[index];  \
-		auto& source = config.gain[index]; \
-                                           \
-		dest[0] = float(source[0]);        \
-		dest[1] = float(source[1]);        \
-		dest[2] = float(source[2]);        \
-		dest[3] = float(source[3]);        \
+#define CONFIG_GAIN(index)                                                          \
+	if (config.gain##index##Dirty) {                                                \
+		auto& dest = source.gains[index];                                           \
+		auto& sourceGain = config.gain[index];                                      \
+                                                                                    \
+		dest[0] = float(sourceGain[0]);                                             \
+		dest[1] = float(sourceGain[1]);                                             \
+		dest[2] = float(sourceGain[2]);                                             \
+		dest[3] = float(sourceGain[3]);                                             \
+                                                                                    \
+		if (dest[0] == 0.f && dest[1] == 0.f && dest[2] == 0.f && dest[3] == 0.f) { \
+			source.enabledMixStages &= ~(1u << index);                              \
+		} else {                                                                    \
+			source.enabledMixStages |= (1u << index);                               \
+		}                                                                           \
 	}
+
 		CONFIG_GAIN(0);
 		CONFIG_GAIN(1);
 		CONFIG_GAIN(2);
@@ -744,5 +756,6 @@ namespace Audio {
 		currentSamples.clear();
 
 		gains.fill({});
+		enabledMixStages = 0;
 	}
 }  // namespace Audio