diff --git a/Source/Core/Core/CoreTiming.cpp b/Source/Core/Core/CoreTiming.cpp
index 9987ac84e1..6893bc2b91 100644
--- a/Source/Core/Core/CoreTiming.cpp
+++ b/Source/Core/Core/CoreTiming.cpp
@@ -433,12 +433,6 @@ void CoreTimingManager::ResetThrottle(s64 cycle)
   m_throttle_deadline = Clock::now();
 }
 
-TimePoint CoreTimingManager::GetCPUTimePoint(s64 cyclesLate) const
-{
-  return TimePoint(std::chrono::duration_cast<DT>(DT_s(m_globals.global_timer - cyclesLate) /
-                                                  m_throttle_clock_per_sec));
-}
-
 bool CoreTimingManager::GetVISkip() const
 {
   return m_throttle_disable_vi_int && g_ActiveConfig.bVISkip && !Core::WantsDeterminism();
diff --git a/Source/Core/Core/CoreTiming.h b/Source/Core/Core/CoreTiming.h
index 23bd4c3239..88635e47ae 100644
--- a/Source/Core/Core/CoreTiming.h
+++ b/Source/Core/Core/CoreTiming.h
@@ -161,8 +161,8 @@ public:
   // May be used from any thread.
   void SleepUntil(TimePoint time_point);
 
-  TimePoint GetCPUTimePoint(s64 cyclesLate) const;  // Used by Dolphin Analytics
-  bool GetVISkip() const;                           // Used By VideoInterface
+  // Used by VideoInterface
+  bool GetVISkip() const;
 
   bool UseSyncOnSkipIdle() const;
 
diff --git a/Source/Core/VideoCommon/PerformanceMetrics.cpp b/Source/Core/VideoCommon/PerformanceMetrics.cpp
index 773dc44383..d9f3eb582b 100644
--- a/Source/Core/VideoCommon/PerformanceMetrics.cpp
+++ b/Source/Core/VideoCommon/PerformanceMetrics.cpp
@@ -4,13 +4,13 @@
 #include "VideoCommon/PerformanceMetrics.h"
 
 #include <algorithm>
-#include <mutex>
 
 #include <imgui.h>
 #include <implot.h>
 
 #include "Core/Config/GraphicsSettings.h"
 #include "Core/CoreTiming.h"
+#include "Core/HW/SystemTimers.h"
 #include "Core/HW/VideoInterface.h"
 #include "Core/System.h"
 #include "VideoCommon/VideoConfig.h"
@@ -25,7 +25,8 @@ void PerformanceMetrics::Reset()
 
   m_time_sleeping = DT::zero();
   m_real_times.fill(Clock::now());
-  m_cpu_times.fill(Core::System::GetInstance().GetCoreTiming().GetCPUTimePoint(0));
+  m_core_ticks.fill(0);
+  m_max_speed = 0;
 }
 
 void PerformanceMetrics::CountFrame()
@@ -40,18 +41,25 @@ void PerformanceMetrics::CountVBlank()
 
 void PerformanceMetrics::CountThrottleSleep(DT sleep)
 {
-  std::unique_lock lock(m_time_lock);
   m_time_sleeping += sleep;
 }
 
-void PerformanceMetrics::CountPerformanceMarker(Core::System& system, s64 cyclesLate)
+void PerformanceMetrics::CountPerformanceMarker(Core::System& system, s64 cycles_late)
 {
-  std::unique_lock lock(m_time_lock);
   m_speed_counter.Count();
 
-  m_real_times[m_time_index] = Clock::now() - m_time_sleeping;
-  m_cpu_times[m_time_index] = system.GetCoreTiming().GetCPUTimePoint(cyclesLate);
-  m_time_index += 1;
+  const auto ticks = system.GetCoreTiming().GetTicks() - cycles_late;
+  const auto real_time = Clock::now() - m_time_sleeping;
+
+  auto& oldest_ticks = m_core_ticks[m_time_index];
+  auto& oldest_time = m_real_times[m_time_index];
+
+  m_max_speed = DT_s(ticks - oldest_ticks) / system.GetSystemTimers().GetTicksPerSecond() /
+                (real_time - oldest_time);
+
+  oldest_ticks = ticks;
+  oldest_time = real_time;
+  ++m_time_index;
 }
 
 double PerformanceMetrics::GetFPS() const
@@ -71,9 +79,7 @@ double PerformanceMetrics::GetSpeed() const
 
 double PerformanceMetrics::GetMaxSpeed() const
 {
-  std::shared_lock lock(m_time_lock);
-  return DT_s(m_cpu_times[u8(m_time_index - 1)] - m_cpu_times[m_time_index]) /
-         DT_s(m_real_times[u8(m_time_index - 1)] - m_real_times[m_time_index]);
+  return m_max_speed;
 }
 
 double PerformanceMetrics::GetLastSpeedDenominator() const
diff --git a/Source/Core/VideoCommon/PerformanceMetrics.h b/Source/Core/VideoCommon/PerformanceMetrics.h
index d1f4211a46..a3a78019a2 100644
--- a/Source/Core/VideoCommon/PerformanceMetrics.h
+++ b/Source/Core/VideoCommon/PerformanceMetrics.h
@@ -4,7 +4,7 @@
 #pragma once
 
 #include <array>
-#include <shared_mutex>
+#include <atomic>
 
 #include "Common/CommonTypes.h"
 #include "VideoCommon/PerformanceTracker.h"
@@ -51,11 +51,10 @@ private:
 
   double m_graph_max_time = 0.0;
 
-  mutable std::shared_mutex m_time_lock;
-
+  std::atomic<double> m_max_speed{};
   u8 m_time_index = 0;
   std::array<TimePoint, 256> m_real_times{};
-  std::array<TimePoint, 256> m_cpu_times{};
+  std::array<u64, 256> m_core_ticks{};
   DT m_time_sleeping{};
 };