Xargon/Duckstation
1
0
Fork 0
mirror of https://github.com/RetroDECK/Duckstation.git synced 2024-09-17 08:02:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Common: Purge ByteStream

Browse source
This commit is contained in:
Stenzek 2024-07-29 16:27:03 +10:00
parent 6ce9e571ed
commit 0321a2cdc4
No known key found for this signature in database
9 changed files with 0 additions and 1952 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/common/CMakeLists.txt
View file

@ -7,8 +7,6 @@ add_library(common
bitfield.h
bitutils.h
build_timestamp.h
byte_stream.cpp
byte_stream.h
crash_handler.cpp
crash_handler.h
dimensional_array.h

1349
src/common/byte_stream.cpp
View file

File diff suppressed because it is too large Load diff

277
src/common/byte_stream.h
View file

@ -1,277 +0,0 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#pragma once
#include "types.h"
#include <memory>
#include <string>
#include <string_view>
#include <vector>
class Error;
// base byte stream creation functions
enum BYTESTREAM_OPEN_MODE
{
BYTESTREAM_OPEN_READ = 1, // open stream for writing
BYTESTREAM_OPEN_WRITE = 2, // open stream for writing
BYTESTREAM_OPEN_APPEND = 4, // seek to the end
BYTESTREAM_OPEN_TRUNCATE = 8, // truncate the file, seek to start
BYTESTREAM_OPEN_CREATE = 16, // if the file does not exist, create it
BYTESTREAM_OPEN_ATOMIC_UPDATE = 64, //
BYTESTREAM_OPEN_SEEKABLE = 128,
BYTESTREAM_OPEN_STREAMED = 256,
};
// forward declarations for implemented classes
class ByteStream;
class MemoryByteStream;
class GrowableMemoryByteStream;
class ReadOnlyMemoryByteStream;
class NullByteStream;
// interface class used by readers, writers, etc.
class ByteStream
{
public:
virtual ~ByteStream() {}
// reads a single byte from the stream.
virtual bool ReadByte(u8* pDestByte) = 0;
// read bytes from this stream. returns the number of bytes read, if this isn't equal to the requested size, an error
// or EOF occurred.
virtual u32 Read(void* pDestination, u32 ByteCount) = 0;
// read bytes from this stream, optionally returning the number of bytes read.
virtual bool Read2(void* pDestination, u32 ByteCount, u32* pNumberOfBytesRead = nullptr) = 0;
// writes a single byte to the stream.
virtual bool WriteByte(u8 SourceByte) = 0;
// write bytes to this stream, returns the number of bytes written. if this isn't equal to the requested size, a
// buffer overflow, or write error occurred.
virtual u32 Write(const void* pSource, u32 ByteCount) = 0;
// write bytes to this stream, optionally returning the number of bytes written.
virtual bool Write2(const void* pSource, u32 ByteCount, u32* pNumberOfBytesWritten = nullptr) = 0;
// seeks to the specified position in the stream
// if seek failed, returns false.
virtual bool SeekAbsolute(u64 Offset) = 0;
virtual bool SeekRelative(s64 Offset) = 0;
virtual bool SeekToEnd() = 0;
// gets the current offset in the stream
virtual u64 GetPosition() const = 0;
// gets the size of the stream
virtual u64 GetSize() const = 0;
// flush any changes to the stream to disk
virtual bool Flush() = 0;
// if the file was opened in atomic update mode, discards any changes made to the file
virtual bool Discard() = 0;
// if the file was opened in atomic update mode, commits the file and replaces the temporary file
virtual bool Commit() = 0;
// state accessors
inline bool InErrorState() const { return m_errorState; }
inline void SetErrorState() { m_errorState = true; }
inline void ClearErrorState() { m_errorState = false; }
bool ReadU8(u8* dest);
bool ReadU16(u16* dest);
bool ReadU32(u32* dest);
bool ReadU64(u64* dest);
bool ReadS8(s8* dest);
bool ReadS16(s16* dest);
bool ReadS32(s32* dest);
bool ReadS64(s64* dest);
bool ReadSizePrefixedString(std::string* dest);
bool WriteU8(u8 dest);
bool WriteU16(u16 dest);
bool WriteU32(u32 dest);
bool WriteU64(u64 dest);
bool WriteS8(s8 dest);
bool WriteS16(s16 dest);
bool WriteS32(s32 dest);
bool WriteS64(s64 dest);
bool WriteSizePrefixedString(std::string_view str);
// base byte stream creation functions
// opens a local file-based stream. fills in error if passed, and returns false if the file cannot be opened.
static std::unique_ptr<ByteStream> OpenFile(const char* FileName, u32 OpenMode, Error* error = nullptr);
// memory byte stream, caller is responsible for management, therefore it can be located on either the stack or on the
// heap.
static std::unique_ptr<MemoryByteStream> CreateMemoryStream(void* pMemory, u32 Size);
// a growable memory byte stream will automatically allocate its own memory if the provided memory is overflowed.
// a "pure heap" buffer, i.e. a buffer completely managed by this implementation, can be created by supplying a NULL
// pointer and initialSize of zero.
static std::unique_ptr<GrowableMemoryByteStream> CreateGrowableMemoryStream(void* pInitialMemory, u32 InitialSize);
static std::unique_ptr<GrowableMemoryByteStream> CreateGrowableMemoryStream();
// readable memory stream
static std::unique_ptr<ReadOnlyMemoryByteStream> CreateReadOnlyMemoryStream(const void* pMemory, u32 Size);
// null memory stream
static std::unique_ptr<NullByteStream> CreateNullStream();
// zstd stream, actually defined in util/zstd_byte_stream.cpp, to avoid common dependency on libzstd
static std::unique_ptr<ByteStream> CreateZstdCompressStream(ByteStream* src_stream, int compression_level);
static std::unique_ptr<ByteStream> CreateZstdDecompressStream(ByteStream* src_stream, u32 compressed_size);
// copies one stream's contents to another. rewinds source streams automatically, and returns it back to its old
// position.
static bool CopyStream(ByteStream* pDestinationStream, ByteStream* pSourceStream);
// appends one stream's contents to another.
static bool AppendStream(ByteStream* pSourceStream, ByteStream* pDestinationStream);
// copies a number of bytes from one to another
static u32 CopyBytes(ByteStream* pSourceStream, u32 byteCount, ByteStream* pDestinationStream);
static std::string ReadStreamToString(ByteStream* stream, bool seek_to_start = true);
static bool WriteStreamToString(std::string_view sv, ByteStream* stream);
static std::vector<u8> ReadBinaryStream(ByteStream* stream, bool seek_to_start = true);
static bool WriteBinaryToStream(ByteStream* stream, const void* data, size_t data_length);
protected:
ByteStream() : m_errorState(false) {}
// state bits
bool m_errorState;
// make it noncopyable
ByteStream(const ByteStream&) = delete;
ByteStream& operator=(const ByteStream&) = delete;
};
class NullByteStream final : public ByteStream
{
public:
NullByteStream();
~NullByteStream() override;
bool ReadByte(u8* pDestByte) override;
u32 Read(void* pDestination, u32 ByteCount) override;
bool Read2(void* pDestination, u32 ByteCount, u32* pNumberOfBytesRead /* = nullptr */) override;
bool WriteByte(u8 SourceByte) override;
u32 Write(const void* pSource, u32 ByteCount) override;
bool Write2(const void* pSource, u32 ByteCount, u32* pNumberOfBytesWritten /* = nullptr */) override;
bool SeekAbsolute(u64 Offset) override;
bool SeekRelative(s64 Offset) override;
bool SeekToEnd() override;
u64 GetSize() const override;
u64 GetPosition() const override;
bool Flush() override;
bool Commit() override;
bool Discard() override;
};
class MemoryByteStream final : public ByteStream
{
public:
MemoryByteStream(void* pMemory, u32 MemSize);
~MemoryByteStream() override;
u8* GetMemoryPointer() const { return m_pMemory; }
u32 GetMemorySize() const { return m_iSize; }
bool ReadByte(u8* pDestByte) override;
u32 Read(void* pDestination, u32 ByteCount) override;
bool Read2(void* pDestination, u32 ByteCount, u32* pNumberOfBytesRead /* = nullptr */) override;
bool WriteByte(u8 SourceByte) override;
u32 Write(const void* pSource, u32 ByteCount) override;
bool Write2(const void* pSource, u32 ByteCount, u32* pNumberOfBytesWritten /* = nullptr */) override;
bool SeekAbsolute(u64 Offset) override;
bool SeekRelative(s64 Offset) override;
bool SeekToEnd() override;
u64 GetSize() const override;
u64 GetPosition() const override;
bool Flush() override;
bool Commit() override;
bool Discard() override;
private:
u8* m_pMemory;
u32 m_iPosition;
u32 m_iSize;
};
class ReadOnlyMemoryByteStream final : public ByteStream
{
public:
ReadOnlyMemoryByteStream(const void* pMemory, u32 MemSize);
~ReadOnlyMemoryByteStream() override;
const u8* GetMemoryPointer() const { return m_pMemory; }
u32 GetMemorySize() const { return m_iSize; }
bool ReadByte(u8* pDestByte) override;
u32 Read(void* pDestination, u32 ByteCount) override;
bool Read2(void* pDestination, u32 ByteCount, u32* pNumberOfBytesRead /* = nullptr */) override;
bool WriteByte(u8 SourceByte) override;
u32 Write(const void* pSource, u32 ByteCount) override;
bool Write2(const void* pSource, u32 ByteCount, u32* pNumberOfBytesWritten /* = nullptr */) override;
bool SeekAbsolute(u64 Offset) override;
bool SeekRelative(s64 Offset) override;
bool SeekToEnd() override;
u64 GetSize() const override;
u64 GetPosition() const override;
bool Flush() override;
bool Commit() override;
bool Discard() override;
private:
const u8* m_pMemory;
u32 m_iPosition;
u32 m_iSize;
};
class GrowableMemoryByteStream final : public ByteStream
{
public:
GrowableMemoryByteStream(void* pInitialMem, u32 InitialMemSize);
~GrowableMemoryByteStream() override;
u8* GetMemoryPointer() const { return m_pMemory; }
u32 GetMemorySize() const { return m_iMemorySize; }
void Resize(u32 new_size);
void ResizeMemory(u32 new_size);
void EnsureSpace(u32 space);
void ShrinkToFit();
bool ReadByte(u8* pDestByte) override;
u32 Read(void* pDestination, u32 ByteCount) override;
bool Read2(void* pDestination, u32 ByteCount, u32* pNumberOfBytesRead /* = nullptr */) override;
bool WriteByte(u8 SourceByte) override;
u32 Write(const void* pSource, u32 ByteCount) override;
bool Write2(const void* pSource, u32 ByteCount, u32* pNumberOfBytesWritten /* = nullptr */) override;
bool SeekAbsolute(u64 Offset) override;
bool SeekRelative(s64 Offset) override;
bool SeekToEnd() override;
u64 GetSize() const override;
u64 GetPosition() const override;
bool Flush() override;
bool Commit() override;
bool Discard() override;
private:
void Grow(u32 MinimumGrowth);
u8* m_pPrivateMemory;
u8* m_pMemory;
u32 m_iPosition;
u32 m_iSize;
u32 m_iMemorySize;
};

2
src/common/common.vcxproj
View file

@ -7,7 +7,6 @@
<ClInclude Include="bitfield.h" />
<ClInclude Include="bitutils.h" />
<ClInclude Include="build_timestamp.h" />
<ClInclude Include="byte_stream.h" />
<ClInclude Include="crash_handler.h" />
<ClInclude Include="dimensional_array.h" />
<ClInclude Include="dynamic_library.h" />
@ -50,7 +49,6 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="assert.cpp" />
<ClCompile Include="byte_stream.cpp" />
<ClCompile Include="crash_handler.cpp" />
<ClCompile Include="dynamic_library.cpp" />
<ClCompile Include="error.cpp" />

2
src/common/common.vcxproj.filters
View file

@ -7,7 +7,6 @@
<ClInclude Include="heap_array.h" />
<ClInclude Include="log.h" />
<ClInclude Include="small_string.h" />
<ClInclude Include="byte_stream.h" />
<ClInclude Include="timer.h" />
<ClInclude Include="assert.h" />
<ClInclude Include="align.h" />
@ -53,7 +52,6 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="small_string.cpp" />
<ClCompile Include="byte_stream.cpp" />
<ClCompile Include="log.cpp" />
<ClCompile Include="timer.cpp" />
<ClCompile Include="assert.cpp" />

1
src/util/CMakeLists.txt
View file

@ -67,7 +67,6 @@ add_library(util
wav_writer.h
window_info.cpp
window_info.h
zstd_byte_stream.cpp
)
target_precompile_headers(util PRIVATE "pch.h")

1
src/util/util.vcxproj
View file

@ -199,7 +199,6 @@
<ClCompile Include="win32_raw_input_source.cpp" />
<ClCompile Include="window_info.cpp" />
<ClCompile Include="xinput_source.cpp" />
<ClCompile Include="zstd_byte_stream.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\dep\cubeb\cubeb.vcxproj">

1
src/util/util.vcxproj.filters
View file

@ -142,7 +142,6 @@
<ClCompile Include="http_downloader.cpp" />
<ClCompile Include="metal_device.mm" />
<ClCompile Include="metal_stream_buffer.mm" />
<ClCompile Include="zstd_byte_stream.cpp" />
<ClCompile Include="opengl_context.cpp" />
<ClCompile Include="opengl_context_egl.cpp" />
<ClCompile Include="opengl_context_egl_wayland.cpp" />

317
src/util/zstd_byte_stream.cpp
View file

@ -1,317 +0,0 @@
// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#include "common/byte_stream.h"
#include "common/log.h"
#include <zstd.h>
#include <zstd_errors.h>
Log_SetChannel(ByteStream);
namespace {
class ZstdCompressStream final : public ByteStream
{
public:
ZstdCompressStream(ByteStream* dst_stream, int compression_level) : m_dst_stream(dst_stream)
{
m_cstream = ZSTD_createCStream();
ZSTD_CCtx_setParameter(m_cstream, ZSTD_c_compressionLevel, compression_level);
}
~ZstdCompressStream() override
{
if (!m_done)
Compress(ZSTD_e_end);
ZSTD_freeCStream(m_cstream);
}
bool ReadByte(u8* pDestByte) override { return false; }
u32 Read(void* pDestination, u32 ByteCount) override { return 0; }
bool Read2(void* pDestination, u32 ByteCount, u32* pNumberOfBytesRead = nullptr) override { return false; }
bool WriteByte(u8 SourceByte) override
{
if (m_input_buffer_wpos == INPUT_BUFFER_SIZE && !Compress(ZSTD_e_continue))
return false;
m_input_buffer[m_input_buffer_wpos++] = SourceByte;
return true;
}
u32 Write(const void* pSource, u32 ByteCount) override
{
u32 remaining = ByteCount;
const u8* read_ptr = static_cast<const u8*>(pSource);
for (;;)
{
const u32 copy_size = std::min(INPUT_BUFFER_SIZE - m_input_buffer_wpos, remaining);
std::memcpy(&m_input_buffer[m_input_buffer_wpos], read_ptr, copy_size);
read_ptr += copy_size;
remaining -= copy_size;
m_input_buffer_wpos += copy_size;
if (remaining == 0 || !Compress(ZSTD_e_continue))
break;
}
return ByteCount - remaining;
}
bool Write2(const void* pSource, u32 ByteCount, u32* pNumberOfBytesWritten = nullptr) override
{
const u32 bytes_written = Write(pSource, ByteCount);
if (pNumberOfBytesWritten)
*pNumberOfBytesWritten = bytes_written;
return (bytes_written == ByteCount);
}
bool SeekAbsolute(u64 Offset) override { return false; }
bool SeekRelative(s64 Offset) override { return (Offset == 0); }
bool SeekToEnd() override { return false; }
u64 GetPosition() const override { return m_position; }
u64 GetSize() const override { return 0; }
bool Flush() override { return Compress(ZSTD_e_flush); }
bool Discard() override { return true; }
bool Commit() override { return Compress(ZSTD_e_end); }
private:
enum : u32
{
INPUT_BUFFER_SIZE = 131072,
OUTPUT_BUFFER_SIZE = 65536,
};
bool Compress(ZSTD_EndDirective action)
{
if (m_errorState || m_done)
return false;
ZSTD_inBuffer inbuf = {m_input_buffer, m_input_buffer_wpos, 0};
for (;;)
{
ZSTD_outBuffer outbuf = {m_output_buffer, OUTPUT_BUFFER_SIZE, 0};
const size_t ret = ZSTD_compressStream2(m_cstream, &outbuf, &inbuf, action);
if (ZSTD_isError(ret))
{
ERROR_LOG("ZSTD_compressStream2() failed: {} ({})", static_cast<unsigned>(ZSTD_getErrorCode(ret)),
ZSTD_getErrorString(ZSTD_getErrorCode(ret)));
SetErrorState();
return false;
}
if (outbuf.pos > 0)
{
if (!m_dst_stream->Write2(m_output_buffer, static_cast<u32>(outbuf.pos)))
{
SetErrorState();
return false;
}
outbuf.pos = 0;
}
if (action == ZSTD_e_end)
{
// break when compression output has finished
if (ret == 0)
{
m_done = true;
break;
}
}
else
{
// break when all input data is consumed
if (inbuf.pos == inbuf.size)
break;
}
}
m_position += m_input_buffer_wpos;
m_input_buffer_wpos = 0;
return true;
}
ByteStream* m_dst_stream;
ZSTD_CStream* m_cstream = nullptr;
u64 m_position = 0;
u32 m_input_buffer_wpos = 0;
bool m_done = false;
u8 m_input_buffer[INPUT_BUFFER_SIZE];
u8 m_output_buffer[OUTPUT_BUFFER_SIZE];
};
} // namespace
std::unique_ptr<ByteStream> ByteStream::CreateZstdCompressStream(ByteStream* src_stream, int compression_level)
{
return std::make_unique<ZstdCompressStream>(src_stream, compression_level);
}
namespace {
class ZstdDecompressStream final : public ByteStream
{
public:
ZstdDecompressStream(ByteStream* src_stream, u32 compressed_size)
: m_src_stream(src_stream), m_bytes_remaining(compressed_size)
{
m_cstream = ZSTD_createDStream();
m_in_buffer.src = m_input_buffer;
Decompress();
}
~ZstdDecompressStream() override { ZSTD_freeDStream(m_cstream); }
bool ReadByte(u8* pDestByte) override { return Read(pDestByte, sizeof(u8)) == sizeof(u8); }
u32 Read(void* pDestination, u32 ByteCount) override
{
u8* write_ptr = static_cast<u8*>(pDestination);
u32 remaining = ByteCount;
for (;;)
{
const u32 copy_size = std::min<u32>(m_output_buffer_wpos - m_output_buffer_rpos, remaining);
std::memcpy(write_ptr, &m_output_buffer[m_output_buffer_rpos], copy_size);
m_output_buffer_rpos += copy_size;
write_ptr += copy_size;
remaining -= copy_size;
if (remaining == 0 || !Decompress())
break;
}
return ByteCount - remaining;
}
bool Read2(void* pDestination, u32 ByteCount, u32* pNumberOfBytesRead = nullptr) override
{
const u32 bytes_read = Read(pDestination, ByteCount);
if (pNumberOfBytesRead)
*pNumberOfBytesRead = bytes_read;
return (bytes_read == ByteCount);
}
bool WriteByte(u8 SourceByte) override { return false; }
u32 Write(const void* pSource, u32 ByteCount) override { return 0; }
bool Write2(const void* pSource, u32 ByteCount, u32* pNumberOfBytesWritten = nullptr) override { return false; }
bool SeekAbsolute(u64 Offset) override { return false; }
bool SeekRelative(s64 Offset) override
{
if (Offset < 0)
return false;
else if (Offset == 0)
return true;
s64 remaining = Offset;
for (;;)
{
const s64 skip = std::min<s64>(m_output_buffer_wpos - m_output_buffer_rpos, remaining);
remaining -= skip;
m_output_buffer_rpos += static_cast<u32>(skip);
if (remaining == 0)
return true;
else if (!Decompress())
return false;
}
}
bool SeekToEnd() override { return false; }
u64 GetPosition() const override { return 0; }
u64 GetSize() const override { return 0; }
bool Flush() override { return true; }
bool Discard() override { return true; }
bool Commit() override { return true; }
private:
enum : u32
{
INPUT_BUFFER_SIZE = 65536,
OUTPUT_BUFFER_SIZE = 131072,
};
bool Decompress()
{
if (m_output_buffer_rpos != m_output_buffer_wpos)
{
const u32 move_size = m_output_buffer_wpos - m_output_buffer_rpos;
std::memmove(&m_output_buffer[0], &m_output_buffer[m_output_buffer_rpos], move_size);
m_output_buffer_rpos = move_size;
m_output_buffer_wpos = move_size;
}
else
{
m_output_buffer_rpos = 0;
m_output_buffer_wpos = 0;
}
ZSTD_outBuffer outbuf = {m_output_buffer, OUTPUT_BUFFER_SIZE - m_output_buffer_wpos, 0};
while (outbuf.pos == 0)
{
if (m_in_buffer.pos == m_in_buffer.size && !m_errorState)
{
const u32 requested_size = std::min<u32>(m_bytes_remaining, INPUT_BUFFER_SIZE);
const u32 bytes_read = m_src_stream->Read(m_input_buffer, requested_size);
m_in_buffer.size = bytes_read;
m_in_buffer.pos = 0;
m_bytes_remaining -= bytes_read;
if (bytes_read != requested_size || m_bytes_remaining == 0)
{
m_errorState = true;
break;
}
}
size_t ret = ZSTD_decompressStream(m_cstream, &outbuf, &m_in_buffer);
if (ZSTD_isError(ret))
{
ERROR_LOG("ZSTD_decompressStream() failed: {} ({})", static_cast<unsigned>(ZSTD_getErrorCode(ret)),
ZSTD_getErrorString(ZSTD_getErrorCode(ret)));
m_in_buffer.pos = m_in_buffer.size;
m_output_buffer_rpos = 0;
m_output_buffer_wpos = 0;
m_errorState = true;
return false;
}
}
m_output_buffer_wpos = static_cast<u32>(outbuf.pos);
return true;
}
ByteStream* m_src_stream;
ZSTD_DStream* m_cstream = nullptr;
ZSTD_inBuffer m_in_buffer = {};
u32 m_output_buffer_rpos = 0;
u32 m_output_buffer_wpos = 0;
u32 m_bytes_remaining;
bool m_errorState = false;
u8 m_input_buffer[INPUT_BUFFER_SIZE];
u8 m_output_buffer[OUTPUT_BUFFER_SIZE];
};
} // namespace
std::unique_ptr<ByteStream> ByteStream::CreateZstdDecompressStream(ByteStream* src_stream, u32 compressed_size)
{
return std::make_unique<ZstdDecompressStream>(src_stream, compressed_size);
}