System: Support compressing save states

src/core/CMakeLists.txt

@@ -117,7 +117,7 @@ set(RECOMPILER_SRCS
 target_include_directories(core PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..")
 target_include_directories(core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/..")
 target_link_libraries(core PUBLIC Threads::Threads common util zlib)
-target_link_libraries(core PRIVATE stb xxhash imgui rapidjson tinyxml2)
+target_link_libraries(core PRIVATE stb xxhash imgui rapidjson tinyxml2 Zstd::Zstd)
 
 if(WIN32)
   target_sources(core PRIVATE

src/core/core.props

@@ -20,7 +20,7 @@
 
   <ItemDefinitionGroup>
     <Lib>
-      <AdditionalDependencies>$(RootBuildDir)tinyxml2\tinyxml2.lib;$(RootBuildDir)rcheevos\rcheevos.lib;$(RootBuildDir)imgui\imgui.lib;$(RootBuildDir)stb\stb.lib;$(RootBuildDir)xxhash\xxhash.lib;$(RootBuildDir)zlib\zlib.lib;$(RootBuildDir)util\util.lib;$(RootBuildDir)common\common.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>$(RootBuildDir)zstd\zstd.lib;$(RootBuildDir)tinyxml2\tinyxml2.lib;$(RootBuildDir)rcheevos\rcheevos.lib;$(RootBuildDir)imgui\imgui.lib;$(RootBuildDir)stb\stb.lib;$(RootBuildDir)xxhash\xxhash.lib;$(RootBuildDir)zlib\zlib.lib;$(RootBuildDir)util\util.lib;$(RootBuildDir)common\common.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalDependencies Condition="('$(BuildingForUWP)'!='true' And '$(Platform)'!='ARM64')">$(RootBuildDir)rainterface\rainterface.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalDependencies Condition="'$(Platform)'=='ARM64'">$(RootBuildDir)vixl\vixl.lib;%(AdditionalDependencies)</AdditionalDependencies>
     </Lib>

src/core/core.vcxproj

@@ -162,6 +162,7 @@
   <ItemDefinitionGroup>
     <ClCompile>
       <ObjectFileName>$(IntDir)/%(RelativeDir)/</ObjectFileName>
+      <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)dep\zstd\lib</AdditionalIncludeDirectories>
     </ClCompile>
   </ItemDefinitionGroup>
   <Import Project="..\..\dep\msvc\vsprops\Targets.props" />

src/core/save_state_version.h

@@ -13,7 +13,11 @@ struct SAVE_STATE_HEADER
   enum : u32
   {
     MAX_TITLE_LENGTH = 128,
-    MAX_GAME_CODE_LENGTH = 32
+    MAX_GAME_CODE_LENGTH = 32,
+
+    COMPRESSION_TYPE_NONE = 0,
+    COMPRESSION_TYPE_ZLIB = 1,
+    COMPRESSION_TYPE_ZSTD = 2,
   };
 
   u32 magic;

src/core/settings.cpp

@@ -167,7 +167,8 @@ void Settings::Load(SettingsInterface& si)
   pause_on_focus_loss = si.GetBoolValue("Main", "PauseOnFocusLoss", false);
   pause_on_menu = si.GetBoolValue("Main", "PauseOnMenu", true);
   save_state_on_exit = si.GetBoolValue("Main", "SaveStateOnExit", true);
-  create_save_state_backups = si.GetBoolValue("Main", "CreateSaveStateBackups", true);
+  create_save_state_backups = si.GetBoolValue("Main", "CreateSaveStateBackups", DEFAULT_SAVE_STATE_BACKUPS);
+  compress_save_states = si.GetBoolValue("Main", "CompressSaveStates", DEFAULT_SAVE_STATE_COMPRESSION);
   confim_power_off = si.GetBoolValue("Main", "ConfirmPowerOff", true);
   load_devices_from_save_states = si.GetBoolValue("Main", "LoadDevicesFromSaveStates", false);
   apply_compatibility_settings = si.GetBoolValue("Main", "ApplyCompatibilitySettings", true);
@@ -391,6 +392,7 @@ void Settings::Save(SettingsInterface& si) const
   si.SetBoolValue("Main", "PauseOnMenu", pause_on_menu);
   si.SetBoolValue("Main", "SaveStateOnExit", save_state_on_exit);
   si.SetBoolValue("Main", "CreateSaveStateBackups", create_save_state_backups);
+  si.SetBoolValue("Main", "CompressSaveStates", compress_save_states);
   si.SetBoolValue("Main", "ConfirmPowerOff", confim_power_off);
   si.SetBoolValue("Main", "LoadDevicesFromSaveStates", load_devices_from_save_states);
   si.SetBoolValue("Main", "ApplyCompatibilitySettings", apply_compatibility_settings);

src/core/settings.h

@@ -70,7 +70,8 @@ struct Settings
   bool pause_on_focus_loss = false;
   bool pause_on_menu = true;
   bool save_state_on_exit = true;
-  bool create_save_state_backups = false;
+  bool create_save_state_backups = DEFAULT_SAVE_STATE_BACKUPS;
+  bool compress_save_states = DEFAULT_SAVE_STATE_COMPRESSION;
   bool confim_power_off = true;
   bool load_devices_from_save_states = false;
   bool apply_compatibility_settings = true;
@@ -434,11 +435,15 @@ struct Settings
 
   // Android doesn't create settings until they're first opened, so we have to override the defaults here.
 #ifndef __ANDROID__
+  static constexpr bool DEFAULT_SAVE_STATE_COMPRESSION = true;
+  static constexpr bool DEFAULT_SAVE_STATE_BACKUPS = true;
   static constexpr bool DEFAULT_VSYNC_VALUE = false;
   static constexpr bool DEFAULT_FAST_BOOT_VALUE = false;
   static constexpr float DEFAULT_DISPLAY_MAX_FPS = 0.0f;
 #else
-  static constexpr bool DEFAULT_VSYNC_VALUE = true;
+  static constexpr bool DEFAULT_SAVE_STATE_COMPRESSION = true;
+  static constexpr bool DEFAULT_SAVE_STATE_BACKUPS = false;
+  static constexpr bool DEFAULT_VSYNC_VALUE = false;
   static constexpr bool DEFAULT_FAST_BOOT_VALUE = true;
   static constexpr float DEFAULT_DISPLAY_MAX_FPS = 60.0f;
 #endif

src/core/system.cpp

@@ -43,6 +43,8 @@
 #include "util/iso_reader.h"
 #include "util/state_wrapper.h"
 #include "xxhash.h"
+#include "zstd.h"
+#include "zstd_errors.h"
 #include <cctype>
 #include <cinttypes>
 #include <cmath>
@@ -80,7 +82,8 @@ struct MemorySaveState
 
 namespace System {
 static std::optional<ExtendedSaveStateInfo> InternalGetExtendedSaveStateInfo(ByteStream* stream);
-static bool InternalSaveState(ByteStream* state, u32 screenshot_size = 256);
+static bool InternalSaveState(ByteStream* state, u32 screenshot_size = 256,
+                              u32 compression_method = SAVE_STATE_HEADER::COMPRESSION_TYPE_NONE);
 static bool SaveMemoryState(MemorySaveState* mss);
 static bool LoadMemoryState(const MemorySaveState& mss);
 
@@ -963,6 +966,8 @@ bool System::LoadState(const char* filename)
   }
 #endif
 
+  Common::Timer load_timer;
+
   std::unique_ptr<ByteStream> stream = ByteStream::OpenFile(filename, BYTESTREAM_OPEN_READ | BYTESTREAM_OPEN_STREAMED);
   if (!stream)
     return false;
@@ -994,6 +999,7 @@ bool System::LoadState(const char* filename)
   ResetPerformanceCounters();
   ResetThrottler();
   Host::RenderDisplay();
+  Log_VerbosePrintf("Loading state took %.2f msec", load_timer.GetTimeMilliseconds());
   return true;
 }
 
@@ -1006,6 +1012,8 @@ bool System::SaveState(const char* filename, bool backup_existing_save)
       Log_ErrorPrintf("Failed to rename save state backup '%s'", backup_filename.c_str());
   }
 
+  Common::Timer save_timer;
+
   std::unique_ptr<ByteStream> stream =
     ByteStream::OpenFile(filename, BYTESTREAM_OPEN_CREATE | BYTESTREAM_OPEN_WRITE | BYTESTREAM_OPEN_TRUNCATE |
                                      BYTESTREAM_OPEN_ATOMIC_UPDATE | BYTESTREAM_OPEN_STREAMED);
@@ -1014,7 +1022,10 @@ bool System::SaveState(const char* filename, bool backup_existing_save)
 
   Log_InfoPrintf("Saving state to '%s'...", filename);
 
-  const bool result = InternalSaveState(stream.get());
+  const u32 screenshot_size = 256;
+  const bool result = InternalSaveState(stream.get(), screenshot_size,
+                                        g_settings.compress_save_states ? SAVE_STATE_HEADER::COMPRESSION_TYPE_ZSTD :
+                                                                          SAVE_STATE_HEADER::COMPRESSION_TYPE_NONE);
   if (!result)
   {
     Host::ReportFormattedErrorAsync(Host::TranslateString("OSDMessage", "Save State"),
@@ -1031,6 +1042,7 @@ bool System::SaveState(const char* filename, bool backup_existing_save)
     stream->Commit();
   }
 
+  Log_VerbosePrintf("Saving state took %.2f msec", save_timer.GetTimeMilliseconds());
   return result;
 }
 
@@ -1468,7 +1480,7 @@ void System::RecreateSystem()
 
   const bool was_paused = System::IsPaused();
   std::unique_ptr<ByteStream> stream = ByteStream::CreateGrowableMemoryStream(nullptr, 8 * 1024);
-  if (!System::InternalSaveState(stream.get(), 0) || !stream->SeekAbsolute(0))
+  if (!System::InternalSaveState(stream.get(), 0, SAVE_STATE_HEADER::COMPRESSION_TYPE_NONE) || !stream->SeekAbsolute(0))
   {
     Host::ReportErrorAsync("Error", "Failed to save state before system recreation. Shutting down.");
     DestroySystem();
@@ -1832,12 +1844,6 @@ bool System::DoLoadState(ByteStream* state, bool force_software_renderer, bool u
   if (g_settings.HasAnyPerGameMemoryCards())
     UpdatePerGameMemoryCards();
 
-  if (header.data_compression_type != 0)
-  {
-    Host::ReportFormattedErrorAsync("Error", "Unknown save state compression type %u", header.data_compression_type);
-    return false;
-  }
-
 #ifdef WITH_CHEEVOS
   // Updating game/loading settings can turn on hardcore mode. Catch this.
   if (Achievements::ChallengeModeActive())
@@ -1852,9 +1858,35 @@ bool System::DoLoadState(ByteStream* state, bool force_software_renderer, bool u
   if (!state->SeekAbsolute(header.offset_to_data))
     return false;
 
-  StateWrapper sw(state, StateWrapper::Mode::Read, header.version);
+  if (header.data_compression_type == SAVE_STATE_HEADER::COMPRESSION_TYPE_NONE)
-  if (!DoState(sw, nullptr, update_display, false))
+  {
+    StateWrapper sw(state, StateWrapper::Mode::Read, header.version);
+    if (!DoState(sw, nullptr, update_display, false))
+      return false;
+  }
+  else if (header.data_compression_type == SAVE_STATE_HEADER::COMPRESSION_TYPE_ZSTD)
+  {
+    std::unique_ptr<u8[]> compressed_buffer(std::make_unique<u8[]>(header.data_compressed_size));
+    std::unique_ptr<u8[]> uncompressed_buffer(std::make_unique<u8[]>(header.data_uncompressed_size));
+    if (!state->Read2(compressed_buffer.get(), header.data_compressed_size))
+      return false;
+
+    const size_t result = ZSTD_decompress(uncompressed_buffer.get(), header.data_uncompressed_size,
+                                          compressed_buffer.get(), header.data_compressed_size);
+    if (ZSTD_isError(result) || result != header.data_uncompressed_size)
+      return false;
+
+    compressed_buffer.reset();
+    ReadOnlyMemoryByteStream uncompressed_stream(uncompressed_buffer.get(), header.data_uncompressed_size);
+    StateWrapper sw(&uncompressed_stream, StateWrapper::Mode::Read, header.version);
+    if (!DoState(sw, nullptr, update_display, false))
+      return false;
+  }
+  else
+  {
+    Host::ReportFormattedErrorAsync("Error", "Unknown save state compression type %u", header.data_compression_type);
     return false;
+  }
 
   if (s_state == State::Starting)
     s_state = State::Running;
@@ -1864,7 +1896,8 @@ bool System::DoLoadState(ByteStream* state, bool force_software_renderer, bool u
   return true;
 }
 
-bool System::InternalSaveState(ByteStream* state, u32 screenshot_size /* = 256 */)
+bool System::InternalSaveState(ByteStream* state, u32 screenshot_size /* = 256 */,
+                               u32 compression_method /* = SAVE_STATE_HEADER::COMPRESSION_TYPE_NONE*/)
 {
   if (IsShutdown())
     return false;
@@ -1944,16 +1977,46 @@ bool System::InternalSaveState(ByteStream* state, u32 screenshot_size /* = 256 *
 
     g_gpu->RestoreGraphicsAPIState();
 
-    StateWrapper sw(state, StateWrapper::Mode::Write, SAVE_STATE_VERSION);
+    header.data_compression_type = compression_method;
-    const bool result = DoState(sw, nullptr, false, false);
+
+    bool result = false;
+    if (compression_method == SAVE_STATE_HEADER::COMPRESSION_TYPE_NONE)
+    {
+      StateWrapper sw(state, StateWrapper::Mode::Write, SAVE_STATE_VERSION);
+      result = DoState(sw, nullptr, false, false);
+      header.data_uncompressed_size = static_cast<u32>(state->GetPosition() - header.offset_to_data);
+    }
+    else if (compression_method == SAVE_STATE_HEADER::COMPRESSION_TYPE_ZSTD)
+    {
+      GrowableMemoryByteStream staging(nullptr, MAX_SAVE_STATE_SIZE);
+      StateWrapper sw(&staging, StateWrapper::Mode::Write, SAVE_STATE_VERSION);
+      result = DoState(sw, nullptr, false, false);
+      if (result)
+      {
+        header.data_uncompressed_size = static_cast<u32>(staging.GetSize());
+
+        const size_t max_compressed_size = ZSTD_compressBound(header.data_uncompressed_size * 2);
+        std::unique_ptr<u8[]> compress_buffer(std::make_unique<u8[]>(max_compressed_size));
+        size_t compress_size = ZSTD_compress(compress_buffer.get(), max_compressed_size, staging.GetMemoryPointer(),
+                                             header.data_uncompressed_size, 0);
+        if (ZSTD_isError(compress_size))
+        {
+          Log_ErrorPrintf("ZSTD_compress() failed: %s", ZSTD_getErrorString(ZSTD_getErrorCode(compress_size)));
+        }
+        else
+        {
+          header.data_compressed_size = static_cast<u32>(compress_size);
+          result = state->Write2(compress_buffer.get(), header.data_compressed_size);
+          Log_DevPrintf("Compressed %u bytes of state to %u bytes with zstd", header.data_uncompressed_size,
+                        header.data_compressed_size);
+        }
+      }
+    }
 
     g_gpu->ResetGraphicsAPIState();
 
     if (!result)
       return false;
-
-    header.data_compression_type = 0;
-    header.data_uncompressed_size = static_cast<u32>(state->GetPosition() - header.offset_to_data);
   }
 
   // re-write header
@@ -2255,7 +2318,8 @@ void System::UpdateDisplaySync()
   const bool video_sync_enabled = ShouldUseVSync();
   const bool syncing_to_host_vsync = (s_syncing_to_host && video_sync_enabled && s_display_all_frames);
   const float max_display_fps = (s_throttler_enabled || s_syncing_to_host) ? 0.0f : g_settings.display_max_fps;
-  Log_VerbosePrintf("Using vsync: %s", video_sync_enabled ? "YES" : "NO", syncing_to_host_vsync ? " (for throttling)" : "");
+  Log_VerbosePrintf("Using vsync: %s", video_sync_enabled ? "YES" : "NO",
+                    syncing_to_host_vsync ? " (for throttling)" : "");
   Log_VerbosePrintf("Max display fps: %f (%s)", max_display_fps,
                     s_display_all_frames ? "displaying all frames" : "skipping displaying frames when needed");
 

src/duckstation-qt/generalsettingswidget.cpp

@@ -1,5 +1,6 @@
 #include "generalsettingswidget.h"
 #include "autoupdaterdialog.h"
+#include "generalsettingswidget.h"
 #include "mainwindow.h"
 #include "qtutils.h"
 #include "scmversion/scmversion.h"
@@ -32,7 +33,9 @@ GeneralSettingsWidget::GeneralSettingsWidget(SettingsDialog* dialog, QWidget* pa
   SettingWidgetBinder::BindWidgetToBoolSetting(sif, m_ui.disableWindowResizing, "Main", "DisableWindowResize", false);
   SettingWidgetBinder::BindWidgetToBoolSetting(sif, m_ui.hideMouseCursor, "Main", "HideCursorInFullscreen", true);
   SettingWidgetBinder::BindWidgetToBoolSetting(sif, m_ui.createSaveStateBackups, "Main", "CreateSaveStateBackups",
-                                               false);
+                                               Settings::DEFAULT_SAVE_STATE_BACKUPS);
+  SettingWidgetBinder::BindWidgetToBoolSetting(sif, m_ui.compressSaveStates, "Main", "CompressSaveStates",
+                                               Settings::DEFAULT_SAVE_STATE_COMPRESSION);
   connect(m_ui.renderToSeparateWindow, &QCheckBox::stateChanged, this,
           &GeneralSettingsWidget::onRenderToSeparateWindowChanged);
 

src/duckstation-qt/generalsettingswidget.ui

@@ -39,6 +39,13 @@
         </property>
        </widget>
       </item>
+      <item row="4" column="0">
+       <widget class="QCheckBox" name="confirmPowerOff">
+        <property name="text">
+         <string>Confirm Power Off</string>
+        </property>
+       </widget>
+      </item>
       <item row="3" column="1">
        <widget class="QCheckBox" name="saveStateOnExit">
         <property name="text">
@@ -46,6 +53,13 @@
         </property>
        </widget>
       </item>
+      <item row="0" column="1">
+       <widget class="QCheckBox" name="pauseOnFocusLoss">
+        <property name="text">
+         <string>Pause On Focus Loss</string>
+        </property>
+       </widget>
+      </item>
       <item row="6" column="0">
        <widget class="QCheckBox" name="applyGameSettings">
         <property name="text">
@@ -53,6 +67,13 @@
         </property>
        </widget>
       </item>
+      <item row="7" column="0">
+       <widget class="QCheckBox" name="createSaveStateBackups">
+        <property name="text">
+         <string>Create Save State Backups</string>
+        </property>
+       </widget>
+      </item>
       <item row="0" column="0">
        <widget class="QCheckBox" name="inhibitScreensaver">
         <property name="text">
@@ -67,20 +88,6 @@
         </property>
        </widget>
       </item>
-      <item row="4" column="0">
-       <widget class="QCheckBox" name="confirmPowerOff">
-        <property name="text">
-         <string>Confirm Power Off</string>
-        </property>
-       </widget>
-      </item>
-      <item row="0" column="1">
-       <widget class="QCheckBox" name="pauseOnFocusLoss">
-        <property name="text">
-         <string>Pause On Focus Loss</string>
-        </property>
-       </widget>
-      </item>
       <item row="3" column="0">
        <widget class="QCheckBox" name="pauseOnStart">
         <property name="text">
@@ -88,17 +95,17 @@
         </property>
        </widget>
       </item>
-      <item row="7" column="0">
+      <item row="8" column="0">
-       <widget class="QCheckBox" name="createSaveStateBackups">
+       <widget class="QCheckBox" name="enableDiscordPresence">
         <property name="text">
-         <string>Create Save State Backups</string>
+         <string>Enable Discord Presence</string>
         </property>
        </widget>
       </item>
       <item row="7" column="1">
-       <widget class="QCheckBox" name="enableDiscordPresence">
+       <widget class="QCheckBox" name="compressSaveStates">
         <property name="text">
-         <string>Enable Discord Presence</string>
+         <string>Compress Save States</string>
         </property>
        </widget>
       </item>