#include "hdd_image.h" #include "YBaseLib/FileSystem.h" #include "YBaseLib/Log.h" Log_SetChannel(HDDImage); #pragma pack(push, 1) static constexpr u32 LOG_FILE_MAGIC = 0x89374897; struct LOG_FILE_HEADER { u32 magic; u32 sector_size; u64 image_size; u32 sector_count; u32 version_number; u8 padding[12]; }; static constexpr u32 STATE_MAGIC = 0x92087348; struct STATE_HEADER { u32 magic; u32 sector_size; u64 image_size; u32 sector_count; u32 version_number; u32 num_sectors_in_state; u8 padding[8]; }; #pragma pack(pop) static String GetLogFileName(const char* base_filename) { return String::FromFormat("%s.log", base_filename); } static u64 GetSectorMapOffset(HDDImage::SectorIndex index) { return sizeof(LOG_FILE_HEADER) + (static_cast(index) * sizeof(HDDImage::SectorIndex)); } HDDImage::HDDImage(const std::string filename, ByteStream* base_stream, ByteStream* log_stream, u64 size, u32 sector_size, u32 sector_count, u32 version_number, LogSectorMap log_sector_map) : m_filename(std::move(filename)), m_base_stream(base_stream), m_log_stream(log_stream), m_image_size(size), m_sector_size(sector_size), m_sector_count(sector_count), m_version_number(version_number), m_log_sector_map(std::move(log_sector_map)) { m_current_sector.data = std::make_unique(sector_size); } HDDImage::~HDDImage() { m_base_stream->Release(); m_log_stream->Release(); } ByteStream* HDDImage::CreateLogFile(const char* filename, bool truncate_existing, bool atomic_update, u64 image_size, u32 sector_size, u32& num_sectors, u32 version_number, LogSectorMap& sector_map) { if (sector_size == 0 || !Common::IsPow2(sector_size) || ((image_size + (sector_size - 1)) / sector_size) >= std::numeric_limits::max()) { return nullptr; } // Fill sector map with zeros. num_sectors = static_cast((image_size + (sector_size - 1)) / sector_size); sector_map.resize(static_cast(num_sectors)); std::fill_n(sector_map.begin(), static_cast(num_sectors), InvalidSectorNumber); u32 open_flags = BYTESTREAM_OPEN_READ | BYTESTREAM_OPEN_WRITE | BYTESTREAM_OPEN_CREATE | BYTESTREAM_OPEN_SEEKABLE; if (truncate_existing) open_flags |= BYTESTREAM_OPEN_TRUNCATE; if (atomic_update) open_flags |= BYTESTREAM_OPEN_ATOMIC_UPDATE; ByteStream* log_stream = FileSystem::OpenFile(filename, open_flags); if (!log_stream) return nullptr; LOG_FILE_HEADER header = {}; header.magic = LOG_FILE_MAGIC; header.sector_size = sector_size; header.image_size = image_size; header.sector_count = static_cast(num_sectors); header.version_number = version_number; // Write header and sector map to the file. if (!log_stream->Write2(&header, sizeof(header)) || !log_stream->Write2(sector_map.data(), static_cast(sizeof(SectorIndex) * sector_map.size()))) { log_stream->Release(); FileSystem::DeleteFile(filename); return nullptr; } // Align the first sector to 4K, so we better utilize the OS's page cache. u64 pos = log_stream->GetPosition(); if (!Common::IsAlignedPow2(pos, sector_size)) { u64 padding_end = Common::AlignUpPow2(pos, sector_size); while (pos < padding_end) { u64 data = 0; u64 size = std::min(padding_end - pos, u64(sizeof(data))); if (!log_stream->Write2(&data, static_cast(size))) { log_stream->Release(); FileSystem::DeleteFile(filename); return nullptr; } pos += size; } } if (!log_stream->Flush()) { log_stream->Release(); FileSystem::DeleteFile(filename); return nullptr; } return log_stream; } ByteStream* HDDImage::OpenLogFile(const char* filename, u64 image_size, u32& sector_size, u32& num_sectors, u32& version_number, LogSectorMap& sector_map) { ByteStream* log_stream = FileSystem::OpenFile(filename, BYTESTREAM_OPEN_READ | BYTESTREAM_OPEN_WRITE | BYTESTREAM_OPEN_SEEKABLE); if (!log_stream) return nullptr; // Read in the image header. LOG_FILE_HEADER header; if (!log_stream->Read2(&header, sizeof(header)) || header.magic != LOG_FILE_MAGIC || header.image_size != image_size || header.sector_size == 0 || !Common::IsPow2(header.sector_size)) { Log_ErrorPrintf("Log file '%s': Invalid header", filename); log_stream->Release(); return nullptr; } num_sectors = static_cast(image_size / header.sector_size); if (num_sectors == 0 || header.sector_count != static_cast(num_sectors)) { Log_ErrorPrintf("Log file '%s': Corrupted header", filename); log_stream->Release(); return nullptr; } // Read in the sector map. sector_size = header.sector_size; version_number = header.version_number; sector_map.resize(num_sectors); if (!log_stream->Read2(sector_map.data(), static_cast(sizeof(SectorIndex) * num_sectors))) { Log_ErrorPrintf("Failed to read sector map from '%s'", filename); log_stream->Release(); return nullptr; } return log_stream; } HDDImage::SectorBuffer& HDDImage::GetSector(SectorIndex sector_index) { if (m_current_sector.sector_number == sector_index) return m_current_sector; // Unload current sector and replace it. ReleaseSector(m_current_sector); LoadSector(m_current_sector, sector_index); return m_current_sector; } void HDDImage::LoadSector(SectorBuffer& buf, SectorIndex sector_index) { Assert(sector_index != InvalidSectorNumber && sector_index < m_log_sector_map.size()); if (m_log_sector_map[sector_index] == InvalidSectorNumber) LoadSectorFromImage(buf, sector_index); else LoadSectorFromLog(buf, sector_index); } std::unique_ptr HDDImage::Create(const char* filename, u64 size_in_bytes, u32 sector_size /*= DefaultReplaySectorSize*/) { String log_filename = GetLogFileName(filename); if (FileSystem::FileExists(filename) || FileSystem::FileExists(log_filename)) return nullptr; ByteStream* base_stream = FileSystem::OpenFile(filename, BYTESTREAM_OPEN_READ | BYTESTREAM_OPEN_WRITE | BYTESTREAM_OPEN_CREATE | BYTESTREAM_OPEN_SEEKABLE); if (!base_stream) return nullptr; // Write zeros to the image file. u64 image_size = 0; while (image_size < size_in_bytes) { u64 data = 0; const u32 to_write = static_cast(std::min(size_in_bytes - image_size, u64(sizeof(data)))); if (!base_stream->Write2(&data, to_write)) { base_stream->Release(); FileSystem::DeleteFile(filename); return nullptr; } image_size += to_write; } // Create the log. u32 sector_count; LogSectorMap sector_map; ByteStream* log_stream = CreateLogFile(log_filename, false, false, image_size, sector_size, sector_count, 0, sector_map); if (!log_stream) { base_stream->Release(); return nullptr; } return std::unique_ptr( new HDDImage(filename, base_stream, log_stream, image_size, sector_size, sector_count, 0, std::move(sector_map))); } std::unique_ptr HDDImage::Open(const char* filename, u32 sector_size /* = DefaultReplaySectorSize */) { ByteStream* base_stream = FileSystem::OpenFile(filename, BYTESTREAM_OPEN_READ | BYTESTREAM_OPEN_WRITE | BYTESTREAM_OPEN_SEEKABLE); if (!base_stream) return nullptr; u64 image_size = base_stream->GetSize(); if (image_size == 0) { base_stream->Release(); return nullptr; } String log_filename = GetLogFileName(filename); u32 sector_count; u32 version_number = 0; LogSectorMap sector_map; ByteStream* log_stream; if (FileSystem::FileExists(log_filename)) { log_stream = OpenLogFile(log_filename, image_size, sector_size, sector_count, version_number, sector_map); if (!log_stream) { Log_ErrorPrintf("Failed to read log file for image '%s'.", filename); base_stream->Release(); return nullptr; } } else { Log_InfoPrintf("Log file not found for image '%s', creating.", filename); log_stream = CreateLogFile(log_filename, false, false, image_size, sector_size, sector_count, version_number, sector_map); if (!log_stream) { Log_ErrorPrintf("Failed to create log file for image '%s'.", filename); base_stream->Release(); return nullptr; } } Log_DevPrintf("Opened image '%s' with log file '%s' (sector size %u)", filename, log_filename.GetCharArray(), sector_size); return std::unique_ptr(new HDDImage(filename, base_stream, log_stream, image_size, sector_size, sector_count, version_number, std::move(sector_map))); } void HDDImage::LoadSectorFromImage(SectorBuffer& buf, SectorIndex sector_index) { if (!m_base_stream->SeekAbsolute(GetFileOffset(sector_index)) || !m_base_stream->Read2(buf.data.get(), m_sector_size)) Panic("Failed to read from base image."); buf.sector_number = sector_index; buf.dirty = false; buf.in_log = false; } void HDDImage::LoadSectorFromLog(SectorBuffer& buf, SectorIndex sector_index) { DebugAssert(sector_index < m_sector_count); const SectorIndex log_sector_index = m_log_sector_map[sector_index]; Assert(log_sector_index != InvalidSectorNumber); if (!m_log_stream->SeekAbsolute(GetFileOffset(log_sector_index)) || !m_log_stream->Read2(buf.data.get(), m_sector_size)) { Panic("Failed to read from log file."); } buf.sector_number = sector_index; buf.dirty = false; buf.in_log = true; } void HDDImage::WriteSectorToLog(SectorBuffer& buf) { DebugAssert(buf.dirty && buf.sector_number < m_sector_count); // Is the sector currently in the log? if (!buf.in_log) { Assert(m_log_sector_map[buf.sector_number] == InvalidSectorNumber); // Need to allocate it in the log file. if (!m_log_stream->SeekToEnd()) Panic("Failed to seek to end of log."); const u64 sector_offset = m_log_stream->GetPosition(); const SectorIndex log_sector_number = static_cast(sector_offset / m_sector_size); Log_DevPrintf("Allocating log sector %u to sector %u", buf.sector_number, log_sector_number); m_log_sector_map[buf.sector_number] = log_sector_number; // Update log sector map in file. if (!m_log_stream->SeekAbsolute(GetSectorMapOffset(buf.sector_number)) || !m_log_stream->Write2(&log_sector_number, sizeof(SectorIndex))) { Panic("Failed to update sector map in log file."); } buf.in_log = true; } // Write to the log. const SectorIndex log_sector_index = m_log_sector_map[buf.sector_number]; Assert(log_sector_index != InvalidSectorNumber); if (!m_log_stream->SeekAbsolute(GetFileOffset(log_sector_index)) || !m_log_stream->Write2(buf.data.get(), m_sector_size)) { Panic("Failed to write sector to log file."); } buf.dirty = false; } void HDDImage::ReleaseSector(SectorBuffer& buf) { // Write it to the log file if it's changed. if (m_current_sector.dirty) WriteSectorToLog(m_current_sector); m_current_sector.sector_number = InvalidSectorNumber; } void HDDImage::ReleaseAllSectors() { if (m_current_sector.sector_number == InvalidSectorNumber) return; ReleaseSector(m_current_sector); } void HDDImage::Read(void* buffer, u64 offset, u32 size) { Assert((offset + size) <= m_image_size); byte* buf = reinterpret_cast(buffer); while (size > 0) { // Find the sector that this offset lives in. const SectorIndex sector_index = static_cast(offset / m_sector_size); const u32 offset_in_sector = static_cast(offset % m_sector_size); const u32 size_to_read = std::min(size, m_sector_size - offset_in_sector); // Load the sector, and read the sub-sector. const SectorBuffer& sec = GetSector(sector_index); std::memcpy(buf, &sec.data[offset_in_sector], size_to_read); buf += size_to_read; offset += size_to_read; size -= size_to_read; } } void HDDImage::Write(const void* buffer, u64 offset, u32 size) { Assert((offset + size) <= m_image_size); const byte* buf = reinterpret_cast(buffer); while (size > 0) { // Find the sector that this offset lives in. const SectorIndex sector_index = static_cast(offset / m_sector_size); const u32 offset_in_sector = static_cast(offset % m_sector_size); const u32 size_to_write = std::min(size, m_sector_size - offset_in_sector); // Load the sector, and update it. SectorBuffer& sec = GetSector(sector_index); std::memcpy(&sec.data[offset_in_sector], buf, size_to_write); sec.dirty = true; buf += size_to_write; offset += size_to_write; size -= size_to_write; } } bool HDDImage::LoadState(ByteStream* stream) { ReleaseAllSectors(); // Read header in from stream. It may not be valid. STATE_HEADER header; if (!stream->Read2(&header, sizeof(header)) || header.magic != STATE_MAGIC || header.image_size != m_image_size || header.sector_size != m_sector_size || header.sector_count != m_sector_count) { Log_ErrorPrintf("Corrupted save state."); return false; } // The version number could have changed, which means we committed since this state was saved. if (header.version_number != m_version_number) { Log_ErrorPrintf("Incorrect version number in save state (%u, should be %u), it is a stale state", header.version_number, m_version_number); return false; } // Okay, everything seems fine. We can now throw away the current log file, and re-write it. LogSectorMap new_sector_map; ByteStream* new_log_stream = CreateLogFile(GetLogFileName(m_filename.c_str()), true, true, m_image_size, m_sector_size, m_sector_count, m_version_number, new_sector_map); if (!new_log_stream) return false; // Write sectors from log. for (u32 i = 0; i < header.num_sectors_in_state; i++) { const SectorIndex log_sector_index = static_cast(new_log_stream->GetPosition() / m_sector_size); SectorIndex sector_index; if (!stream->Read2(§or_index, sizeof(sector_index)) || sector_index >= m_sector_count || !ByteStream_CopyBytes(stream, m_sector_size, new_log_stream)) { Log_ErrorPrintf("Failed to copy new sector from save state."); new_log_stream->Discard(); new_log_stream->Release(); return false; } // Update new sector map. new_sector_map[sector_index] = log_sector_index; } // Write the new sector map. if (!new_log_stream->SeekAbsolute(GetSectorMapOffset(0)) || !new_log_stream->Write2(new_sector_map.data(), sizeof(SectorIndex) * m_sector_count)) { Log_ErrorPrintf("Failed to write new sector map from save state."); new_log_stream->Discard(); new_log_stream->Release(); return false; } // Commit the stream, replacing the existing file. Then swap the pointers, since we may as well use the existing one. m_log_stream->Release(); new_log_stream->Flush(); new_log_stream->Commit(); m_log_stream = new_log_stream; m_log_sector_map = std::move(new_sector_map); return true; } bool HDDImage::SaveState(ByteStream* stream) { ReleaseAllSectors(); // Precompute how many sectors are committed to the log. u32 log_sector_count = 0; for (SectorIndex sector_index = 0; sector_index < m_sector_count; sector_index++) { if (IsSectorInLog(sector_index)) log_sector_count++; } // Construct header. STATE_HEADER header = {}; header.magic = STATE_MAGIC; header.sector_size = m_sector_size; header.image_size = m_image_size; header.sector_count = m_sector_count; header.version_number = m_version_number; header.num_sectors_in_state = log_sector_count; if (!stream->Write2(&header, sizeof(header))) { Log_ErrorPrintf("Failed to write log header to save state."); return false; } // Copy each sector from the replay log. for (SectorIndex sector_index = 0; sector_index < m_sector_count; sector_index++) { if (!IsSectorInLog(sector_index)) continue; if (!m_log_stream->SeekAbsolute(GetFileOffset(m_log_sector_map[sector_index])) || !stream->Write2(§or_index, sizeof(sector_index)) || !ByteStream_CopyBytes(m_log_stream, m_sector_size, stream)) { Log_ErrorPrintf("Failed to write log sector to save state."); return false; } } return true; } void HDDImage::Flush() { if (!m_current_sector.dirty) return; WriteSectorToLog(m_current_sector); // Ensure the stream isn't buffering. if (!m_log_stream->Flush()) Panic("Failed to flush log stream."); } void HDDImage::CommitLog() { Log_InfoPrintf("Committing log for '%s'.", m_filename.c_str()); ReleaseAllSectors(); for (SectorIndex sector_index = 0; sector_index < m_sector_count; sector_index++) { if (!IsSectorInLog(sector_index)) continue; // Read log sector to buffer, then write it to the base image. // No need to update the log map, since we trash it anyway. if (!m_log_stream->SeekAbsolute(GetFileOffset(m_log_sector_map[sector_index])) || !m_base_stream->SeekAbsolute(GetFileOffset(sector_index)) || ByteStream_CopyBytes(m_log_stream, m_sector_size, m_base_stream) != m_sector_size) { Panic("Failed to transfer sector from log to base image."); } } // Increment the version number, to invalidate old save states. m_version_number++; // Truncate the log, and re-create it. m_log_stream->Release(); m_log_stream = CreateLogFile(GetLogFileName(m_filename.c_str()), true, false, m_image_size, m_sector_size, m_sector_count, m_version_number, m_log_sector_map); } void HDDImage::RevertLog() { Log_InfoPrintf("Reverting log for '%s'", m_filename.c_str()); ReleaseAllSectors(); m_log_stream->Release(); m_log_stream = CreateLogFile(GetLogFileName(m_filename.c_str()), true, false, m_image_size, m_sector_size, m_sector_count, m_version_number, m_log_sector_map); if (!m_log_stream) { Log_ErrorPrintf("Failed to recreate log file for image '%s'", m_filename.c_str()); Panic("Failed to recreate log file."); } }