diff --git a/src/common/audio_stream.cpp b/src/common/audio_stream.cpp
index d4e0b4e43..0b35f58a9 100644
--- a/src/common/audio_stream.cpp
+++ b/src/common/audio_stream.cpp
@@ -170,12 +170,14 @@ void AudioStream::ReadFrames(SampleType* samples, u32 num_frames, bool apply_vol
       }
 
       Log_DevPrintf("Audio buffer underflow, resampled %u frames to %u", samples_copied / m_channels, num_frames);
+      m_underflow_flag.store(true);
     }
     else
     {
       // read nothing, so zero-fill
       std::memset(samples, 0, sizeof(SampleType) * total_samples);
       Log_DevPrintf("Audio buffer underflow with no samples, added %u frames silence", num_frames);
+      m_underflow_flag.store(true);
     }
   }
 
@@ -218,4 +220,5 @@ void AudioStream::EmptyBuffers()
 {
   std::unique_lock<std::mutex> lock(m_buffer_mutex);
   m_buffer.Clear();
+  m_underflow_flag.store(false);
 }
diff --git a/src/common/audio_stream.h b/src/common/audio_stream.h
index 5e7c9b702..e7887792e 100644
--- a/src/common/audio_stream.h
+++ b/src/common/audio_stream.h
@@ -1,6 +1,7 @@
 #pragma once
 #include "fifo_queue.h"
 #include "types.h"
+#include <atomic>
 #include <condition_variable>
 #include <memory>
 #include <mutex>
@@ -45,6 +46,12 @@ public:
   void WriteFrames(const SampleType* frames, u32 num_frames);
   void EndWrite(u32 num_frames);
 
+  bool DidUnderflow()
+  {
+    bool expected = true;
+    return m_underflow_flag.compare_exchange_strong(expected, false);
+  }
+
   static std::unique_ptr<AudioStream> CreateNullAudioStream();
 
   // Latency computation - returns values in seconds
@@ -84,6 +91,7 @@ private:
   mutable std::mutex m_buffer_mutex;
   std::condition_variable m_buffer_draining_cv;
   std::vector<SampleType> m_resample_buffer;
+  std::atomic_bool m_underflow_flag{false};
   u32 m_max_samples = 0;
 
   bool m_output_paused = true;
diff --git a/src/core/system.cpp b/src/core/system.cpp
index ecbf39982..1068b5c4f 100644
--- a/src/core/system.cpp
+++ b/src/core/system.cpp
@@ -1216,8 +1216,22 @@ void UpdateThrottlePeriod()
     static_cast<s32>(1000000000.0 / static_cast<double>(s_throttle_frequency) / static_cast<double>(s_target_speed));
 }
 
+void ResetThrottler()
+{
+  s_last_throttle_time = 0;
+  s_throttle_timer.Reset();
+}
+
 void Throttle()
 {
+  // Reset the throttler on audio buffer overflow, so we don't end up out of phase.
+  if (g_host_interface->GetAudioStream()->DidUnderflow())
+  {
+    Log_DevPrintf("Audio buffer underflowed, resetting throttler");
+    ResetThrottler();
+    return;
+  }
+
   // Allow variance of up to 40ms either way.
   constexpr s64 MAX_VARIANCE_TIME = INT64_C(40000000);
 
@@ -1239,8 +1253,7 @@ void Throttle()
       s_speed_lost_time_timestamp.Reset();
     }
 #endif
-    s_last_throttle_time = 0;
-    s_throttle_timer.Reset();
+    ResetThrottler();
   }
   else if (sleep_time >= MINIMUM_SLEEP_TIME)
   {
@@ -1294,8 +1307,7 @@ void ResetPerformanceCounters()
   s_average_frame_time_accumulator = 0.0f;
   s_worst_frame_time_accumulator = 0.0f;
   s_fps_timer.Reset();
-  s_throttle_timer.Reset();
-  s_last_throttle_time = 0;
+  ResetThrottler();
 }
 
 bool LoadEXE(const char* filename)
diff --git a/src/core/system.h b/src/core/system.h
index 85345ebfc..17d97f787 100644
--- a/src/core/system.h
+++ b/src/core/system.h
@@ -154,6 +154,7 @@ void SetThrottleFrequency(float frequency);
 
 /// Updates the throttle period, call when target emulation speed changes.
 void UpdateThrottlePeriod();
+void ResetThrottler();
 
 /// Throttles the system, i.e. sleeps until it's time to execute the next frame.
 void Throttle();