System: Gracefully handle memory allocaion failure

src/common/memmap.cpp

@@ -1,10 +1,12 @@
-// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "memmap.h"
 #include "align.h"
 #include "assert.h"
+#include "error.h"
 #include "log.h"
+#include "small_string.h"
 #include "string_util.h"
 
 #include "fmt/format.h"
@@ -47,11 +49,15 @@ std::string MemMap::GetFileMappingName(const char* prefix)
   return fmt::format("{}_{}", prefix, pid);
 }
 
-void* MemMap::CreateSharedMemory(const char* name, size_t size)
+void* MemMap::CreateSharedMemory(const char* name, size_t size, Error* error)
 {
-  return static_cast<void*>(CreateFileMappingW(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
-                                               static_cast<DWORD>(size >> 32), static_cast<DWORD>(size),
-                                               StringUtil::UTF8StringToWideString(name).c_str()));
+  const HANDLE mapping =
+    static_cast<void*>(CreateFileMappingW(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, static_cast<DWORD>(size >> 32),
+                                          static_cast<DWORD>(size), StringUtil::UTF8StringToWideString(name).c_str()));
+  if (!mapping)
+    Error::SetWin32(error, "CreateFileMappingW() failed: ", GetLastError());
+
+  return mapping;
 }
 
 void MemMap::DestroySharedMemory(void* ptr)
@@ -299,24 +305,33 @@ std::string MemMap::GetFileMappingName(const char* prefix)
 #endif
 }
 
-void* MemMap::CreateSharedMemory(const char* name, size_t size)
+void* MemMap::CreateSharedMemory(const char* name, size_t size, Error* error)
 {
   const int fd = shm_open(name, O_CREAT | O_EXCL | O_RDWR, 0600);
   if (fd < 0)
   {
-    Log_ErrorPrintf("shm_open failed: %d\n", errno);
+    Error::SetErrno(error, "shm_open failed: ", errno);
     return nullptr;
   }
 
   // we're not going to be opening this mapping in other processes, so remove the file
   shm_unlink(name);
 
+  // use fallocate() to ensure we don't SIGBUS later on.
+#ifdef __linux__
+  if (fallocate(fd, 0, 0, static_cast<off_t>(size)) < 0)
+  {
+    Error::SetErrno(error, TinyString::from_format("fallocate({}) failed: ", size), errno);
+    return nullptr;
+  }
+#else
   // ensure it's the correct size
   if (ftruncate(fd, static_cast<off_t>(size)) < 0)
   {
-    Log_ErrorPrintf("ftruncate(%zu) failed: %d\n", size, errno);
+    Error::SetErrno(error, TinyString::from_format("ftruncate({}) failed: ", size), errno);
     return nullptr;
   }
+#endif
 
   return reinterpret_cast<void*>(static_cast<intptr_t>(fd));
 }

src/common/memmap.h

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #pragma once
@@ -34,9 +34,11 @@ enum class PageProtect : u32
 
 #endif
 
+class Error;
+
 namespace MemMap {
 std::string GetFileMappingName(const char* prefix);
-void* CreateSharedMemory(const char* name, size_t size);
+void* CreateSharedMemory(const char* name, size_t size, Error* error);
 void DestroySharedMemory(void* ptr);
 void* MapSharedMemory(void* handle, size_t offset, void* baseaddr, size_t size, PageProtect mode);
 void UnmapSharedMemory(void* baseaddr, size_t size);

src/core/bus.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "bus.h"
@@ -24,6 +24,7 @@
 
 #include "common/align.h"
 #include "common/assert.h"
+#include "common/error.h"
 #include "common/intrin.h"
 #include "common/log.h"
 #include "common/memmap.h"
@@ -175,10 +176,19 @@ static constexpr size_t TOTAL_SIZE = LUT_OFFSET + LUT_SIZE;
 
 bool Bus::AllocateMemory()
 {
-  s_shmem_handle = MemMap::CreateSharedMemory(MemMap::GetFileMappingName("duckstation").c_str(), MemoryMap::TOTAL_SIZE);
+  Error error;
+  s_shmem_handle =
+    MemMap::CreateSharedMemory(MemMap::GetFileMappingName("duckstation").c_str(), MemoryMap::TOTAL_SIZE, &error);
   if (!s_shmem_handle)
   {
-    Host::ReportErrorAsync("Error", "Failed to allocate memory");
+#ifndef __linux__
+    error.AddSuffix("\nYou may need to close some programs to free up additional memory.");
+#else
+    error.AddSuffix(
+      "\nYou may need to close some programs to free up additional memory, or increase the size of /dev/shm.");
+#endif
+
+    Host::ReportFatalError("Memory Allocation Failed", error.GetDescription());
     return false;
   }
 
@@ -188,7 +198,7 @@ bool Bus::AllocateMemory()
                                                                MemoryMap::RAM_SIZE, PageProtect::ReadWrite));
   if (!g_ram || !g_unprotected_ram)
   {
-    Host::ReportErrorAsync("Error", "Failed to map memory for RAM");
+    Host::ReportFatalError("Memory Allocation Failed", "Failed to map memory for RAM");
     ReleaseMemory();
     return false;
   }
@@ -199,7 +209,7 @@ bool Bus::AllocateMemory()
                                                     MemoryMap::BIOS_SIZE, PageProtect::ReadWrite));
   if (!g_bios)
   {
-    Host::ReportErrorAsync("Error", "Failed to map memory for BIOS");
+    Host::ReportFatalError("Memory Allocation Failed", "Failed to map memory for BIOS");
     ReleaseMemory();
     return false;
   }
@@ -210,7 +220,7 @@ bool Bus::AllocateMemory()
                                                                   MemoryMap::LUT_SIZE, PageProtect::ReadWrite));
   if (!g_memory_handlers)
   {
-    Host::ReportErrorAsync("Error", "Failed to map memory for LUTs");
+    Host::ReportFatalError("Memory Allocation Failed", "Failed to map memory for LUTs");
     ReleaseMemory();
     return false;
   }
@@ -223,7 +233,7 @@ bool Bus::AllocateMemory()
   if (!s_fastmem_arena.Create(FASTMEM_ARENA_SIZE))
   {
     // TODO: maybe make this non-fatal?
-    Host::ReportErrorAsync("Error", "Failed to create fastmem arena");
+    Host::ReportFatalError("Memory Allocation Failed", "Failed to create fastmem arena");
     ReleaseMemory();
     return false;
   }

src/core/bus.h

@@ -1,9 +1,12 @@
-// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #pragma once
-#include "common/bitfield.h"
+
 #include "types.h"
+
+#include "common/bitfield.h"
+
 #include <array>
 #include <bitset>
 #include <optional>

src/core/cpu_code_cache.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #include "bus.h"
@@ -7,11 +7,13 @@
 #include "cpu_core_private.h"
 #include "cpu_disasm.h"
 #include "cpu_recompiler_types.h"
+#include "host.h"
 #include "settings.h"
 #include "system.h"
 #include "timing_event.h"
 
 #include "common/assert.h"
+#include "common/error.h"
 #include "common/intrin.h"
 #include "common/log.h"
 #include "common/memmap.h"
@@ -162,7 +164,7 @@ bool CPU::CodeCache::IsUsingFastmem()
   return IsUsingAnyRecompiler() && g_settings.cpu_fastmem_mode != CPUFastmemMode::Disabled;
 }
 
-void CPU::CodeCache::ProcessStartup()
+bool CPU::CodeCache::ProcessStartup()
 {
   AllocateLUTs();
 
@@ -175,12 +177,18 @@ void CPU::CodeCache::ProcessStartup()
 #endif
   if (!has_buffer && !s_code_buffer.Allocate(RECOMPILER_CODE_CACHE_SIZE, RECOMPILER_FAR_CODE_CACHE_SIZE))
   {
-    Panic("Failed to initialize code space");
+    Host::ReportFatalError("Error", "Failed to initialize code space");
+    return false;
   }
 #endif
 
   if (!Common::PageFaultHandler::InstallHandler(ExceptionHandler))
-    Panic("Failed to install page fault handler");
+  {
+    Host::ReportFatalError("Error", "Failed to install page fault handler");
+    return false;
+  }
+
+  return true;
 }
 
 void CPU::CodeCache::ProcessShutdown()

src/core/cpu_code_cache.h

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #pragma once
@@ -15,7 +15,7 @@ bool IsUsingAnyRecompiler();
 bool IsUsingFastmem();
 
 /// Allocates resources, call once at startup.
-void ProcessStartup();
+bool ProcessStartup();
 
 /// Frees resources, call once at shutdown.
 void ProcessShutdown();

src/core/system.cpp

@@ -242,12 +242,13 @@ static TinyString GetTimestampStringForFileName()
   return TinyString::from_format("{:%Y-%m-%d_%H-%M-%S}", fmt::localtime(std::time(nullptr)));
 }
 
-void System::Internal::ProcessStartup()
+bool System::Internal::ProcessStartup()
 {
   if (!Bus::AllocateMemory())
-    Panic("Failed to allocate memory for emulated bus.");
+    return false;
 
-  CPU::CodeCache::ProcessStartup();
+  if (!CPU::CodeCache::ProcessStartup())
+    return false;
 
   // This will call back to Host::LoadSettings() -> ReloadSources().
   LoadSettings(false);
@@ -263,6 +264,8 @@ void System::Internal::ProcessStartup()
   if (g_settings.enable_discord_presence)
     InitializeDiscordPresence();
 #endif
+
+return true;
 }
 
 void System::Internal::ProcessShutdown()

src/core/system.h

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
 
 #pragma once
@@ -12,6 +12,7 @@
 
 class ByteStream;
 class CDImage;
+class Error;
 class StateWrapper;
 
 class Controller;
@@ -485,7 +486,7 @@ void UpdateDiscordPresence(bool update_session_time);
 
 namespace Internal {
 /// Called on process startup.
-void ProcessStartup();
+bool ProcessStartup();
 
 /// Called on process shutdown.
 void ProcessShutdown();

src/duckstation-nogui/nogui_host.cpp

@@ -30,6 +30,7 @@
 #include "common/assert.h"
 #include "common/byte_stream.h"
 #include "common/crash_handler.h"
+#include "common/error.h"
 #include "common/file_system.h"
 #include "common/log.h"
 #include "common/path.h"
@@ -676,7 +677,11 @@ void NoGUIHost::CPUThreadEntryPoint()
   Threading::SetNameOfCurrentThread("CPU Thread");
 
   // input source setup must happen on emu thread
-  System::Internal::ProcessStartup();
+  if (!System::Internal::ProcessStartup())
+  {
+    g_nogui_window->QuitMessageLoop();
+    return;
+  }
 
   // start the fullscreen UI and get it going
   if (Host::CreateGPUDevice(Settings::GetRenderAPIForRenderer(g_settings.gpu_renderer)) && FullscreenUI::Initialize())

src/duckstation-qt/qthost.cpp

@@ -1497,7 +1497,11 @@ void EmuThread::run()
   m_started_semaphore.release();
 
   // input source setup must happen on emu thread
-  System::Internal::ProcessStartup();
+  if (!System::Internal::ProcessStartup())
+  {
+    moveToThread(m_ui_thread);
+    return;
+  }
 
   // bind buttons/axises
   createBackgroundControllerPollTimer();

src/duckstation-regtest/regtest_host.cpp

@@ -648,18 +648,20 @@ int main(int argc, char* argv[])
 
   if (!autoboot || autoboot->filename.empty())
   {
-    Log_ErrorPrintf("No boot path specified.");
+    Log_ErrorPrint("No boot path specified.");
     return EXIT_FAILURE;
   }
 
-  System::Internal::ProcessStartup();
+  if (!System::Internal::ProcessStartup())
+    return EXIT_FAILURE;
+
   RegTestHost::HookSignals();
 
   int result = -1;
   Log_InfoPrintf("Trying to boot '%s'...", autoboot->filename.c_str());
   if (!System::BootSystem(std::move(autoboot.value())))
   {
-    Log_ErrorPrintf("Failed to boot system.");
+    Log_ErrorPrint("Failed to boot system.");
     goto cleanup;
   }