diff --git a/src/duckstation-qt/displaywidget.cpp b/src/duckstation-qt/displaywidget.cpp
index 9cafa8abb..7feb177fe 100644
--- a/src/duckstation-qt/displaywidget.cpp
+++ b/src/duckstation-qt/displaywidget.cpp
@@ -292,7 +292,7 @@ bool DisplayWidget::event(QEvent* event)
     case QEvent::MouseButtonRelease:
     {
       const u32 button_index = CountTrailingZeros(static_cast<u32>(static_cast<const QMouseEvent*>(event)->button()));
-      emit windowMouseButtonEvent(static_cast<int>(button_index + 1u), event->type() != QEvent::MouseButtonRelease);
+      emit windowMouseButtonEvent(static_cast<int>(button_index), event->type() != QEvent::MouseButtonRelease);
 
       // don't toggle fullscreen when we're bound.. that wouldn't end well.
       if (event->type() == QEvent::MouseButtonDblClick &&
diff --git a/src/frontend-common/imgui_manager.cpp b/src/frontend-common/imgui_manager.cpp
index f3624d322..f5e427dff 100644
--- a/src/frontend-common/imgui_manager.cpp
+++ b/src/frontend-common/imgui_manager.cpp
@@ -745,11 +745,11 @@ void ImGuiManager::UpdateMousePosition(float x, float y)
 
 bool ImGuiManager::ProcessPointerButtonEvent(InputBindingKey key, float value)
 {
-  if (!ImGui::GetCurrentContext() || (key.data - 1u) >= std::size(ImGui::GetIO().MouseDown))
+  if (!ImGui::GetCurrentContext() || key.data >= std::size(ImGui::GetIO().MouseDown))
     return false;
 
   // still update state anyway
-  ImGui::GetIO().AddMouseButtonEvent(key.data - 1, value != 0.0f);
+  ImGui::GetIO().AddMouseButtonEvent(key.data, value != 0.0f);
 
   return s_imgui_wants_mouse.load(std::memory_order_acquire);
 }