From b69c013ec737dc2994bb619dd40595a582fc7660 Mon Sep 17 00:00:00 2001 From: Ovahlord <18347559+Ovahlord@users.noreply.github.com> Date: Fri, 31 Jul 2026 18:45:15 +0200 Subject: [PATCH 1/4] Reduce the amount of unnecessary heap allocations while parsing string tables and fields --- DBCD.Benchmark/Benchmarks/StringTableBench.cs | 41 ++++++++---- DBCD.IO/Extensions.cs | 64 +++++++++++-------- 2 files changed, 66 insertions(+), 39 deletions(-) diff --git a/DBCD.Benchmark/Benchmarks/StringTableBench.cs b/DBCD.Benchmark/Benchmarks/StringTableBench.cs index ad2b760..159e485 100644 --- a/DBCD.Benchmark/Benchmarks/StringTableBench.cs +++ b/DBCD.Benchmark/Benchmarks/StringTableBench.cs @@ -50,27 +50,40 @@ public static string ReadCString(this BinaryReader reader) public static Dictionary ReadStringTable(this BinaryReader reader, int stringTableSize, int baseOffset = 0, bool usePos = false) { - var StringTable = new Dictionary(stringTableSize / 0x20); - if (stringTableSize == 0) - return StringTable; + return []; + + var stringTable = new Dictionary(stringTableSize / 0x20); - var curOfs = 0; - var decoded = Encoding.UTF8.GetString(reader.ReadBytes(stringTableSize)); - foreach (var str in decoded.Split('\0')) + Span stringTableBytes = stackalloc byte[stringTableSize]; + _ = reader.Read(stringTableBytes); + + int start = 0; + for (int i = 0; i < stringTableBytes.Length; ++i) { - if (curOfs == stringTableSize) - break; - + if (stringTableBytes[i] == 0) + { + string str = Encoding.UTF8.GetString(stringTableBytes.Slice(start, i - start)); + if (usePos) + stringTable[reader.BaseStream.Position - stringTableSize + start] = str; + else + stringTable[baseOffset + start] = str; + + start = i + 1; + } + } + + // Trailing string + if (start < stringTableBytes.Length) + { + string str = Encoding.UTF8.GetString(stringTableBytes.Slice(start)); if (usePos) - StringTable[(reader.BaseStream.Position - stringTableSize) + curOfs] = str; + stringTable[reader.BaseStream.Position - stringTableSize + start] = str; else - StringTable[baseOffset + curOfs] = str; - - curOfs += Encoding.UTF8.GetByteCount(str) + 1; + stringTable[baseOffset + start] = str; } - return StringTable; + return stringTable; } } } diff --git a/DBCD.IO/Extensions.cs b/DBCD.IO/Extensions.cs index b26a65c..b72c0b8 100644 --- a/DBCD.IO/Extensions.cs +++ b/DBCD.IO/Extensions.cs @@ -6,6 +6,7 @@ using System.Linq.Expressions; using System.Reflection; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Text; namespace DBCD.IO @@ -50,11 +51,11 @@ public static FieldCache[] ToFieldCache(this Type type) public static T Read(this BinaryReader reader) where T : struct { - byte[] result = reader.ReadBytes(Unsafe.SizeOf()); + Span result = stackalloc byte[Unsafe.SizeOf()]; + _ = reader.Read(result); return Unsafe.ReadUnaligned(ref result[0]); } - /// /// Reads a NUL-separated string table from the current stream /// @@ -64,45 +65,58 @@ public static T Read(this BinaryReader reader) where T : struct /// Base offset to use for the string table keys public static Dictionary ReadStringTable(this BinaryReader reader, int stringTableSize, int baseOffset = 0, bool usePos = false) { - var StringTable = new Dictionary(stringTableSize / 0x20); - - if(stringTableSize == 0) - return StringTable; + if (stringTableSize == 0) + return []; + + var stringTable = new Dictionary(stringTableSize / 0x20); - var curOfs = 0; - var decoded = Encoding.UTF8.GetString(reader.ReadBytes(stringTableSize)); - foreach (var str in decoded.Split('\0')) + Span stringTableBytes = stackalloc byte[stringTableSize]; + _ = reader.Read(stringTableBytes); + + int start = 0; + for (int i = 0; i < stringTableBytes.Length; ++i) { - if (curOfs == stringTableSize) - break; - - if(usePos) - StringTable[(reader.BaseStream.Position - stringTableSize) + curOfs] = str; + if (stringTableBytes[i] == 0) + { + string str = Encoding.UTF8.GetString(stringTableBytes.Slice(start, i - start)); + if (usePos) + stringTable[reader.BaseStream.Position - stringTableSize + start] = str; + else + stringTable[baseOffset + start] = str; + + start = i + 1; + } + } + + // Trailing string + if (start < stringTableBytes.Length) + { + string str = Encoding.UTF8.GetString(stringTableBytes.Slice(start)); + if (usePos) + stringTable[reader.BaseStream.Position - stringTableSize + start] = str; else - StringTable[baseOffset + curOfs] = str; - - curOfs += Encoding.UTF8.GetByteCount(str) + 1; + stringTable[baseOffset + start] = str; } - return StringTable; + return stringTable; } public static T[] ReadArray(this BinaryReader reader) where T : struct { int numBytes = (int)reader.ReadInt64(); - - byte[] result = reader.ReadBytes(numBytes); + Span result = stackalloc byte[numBytes]; + _ = reader.Read(result); reader.BaseStream.Position += (0 - numBytes) & 0x07; - return result.CopyTo(); + return MemoryMarshal.Cast(result).ToArray(); } public static T[] ReadArray(this BinaryReader reader, int size) where T : struct { int numBytes = Unsafe.SizeOf() * size; - - byte[] result = reader.ReadBytes(numBytes); - return result.CopyTo(); + Span result = stackalloc byte[numBytes]; + _ = reader.Read(result); + return MemoryMarshal.Cast(result).ToArray(); } public static unsafe T[] CopyTo(this byte[] src) where T : struct @@ -171,7 +185,7 @@ public static string ReadCString(this BinaryReader reader) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static string ReadCString(this BinaryReader reader, Encoding encoding) { - var bytes = new System.Collections.Generic.List(0x20); + var bytes = new List(0x20); byte b; while ((b = reader.ReadByte()) != 0) bytes.Add(b); From 745e2628e571268874aad8603c91e170d898c212 Mon Sep 17 00:00:00 2001 From: Ovahlord <18347559+Ovahlord@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:44:04 +0200 Subject: [PATCH 2/4] Refactor string table loading to use the array pool instead of stackalloc to prevent possible stack overflows when parsing gigantic amounts of strings at once --- DBCD.Benchmark/Benchmarks/StringTableBench.cs | 52 +++++++++++-------- DBCD.IO/Extensions.cs | 47 ++++++++++------- 2 files changed, 59 insertions(+), 40 deletions(-) diff --git a/DBCD.Benchmark/Benchmarks/StringTableBench.cs b/DBCD.Benchmark/Benchmarks/StringTableBench.cs index 159e485..fca7c9c 100644 --- a/DBCD.Benchmark/Benchmarks/StringTableBench.cs +++ b/DBCD.Benchmark/Benchmarks/StringTableBench.cs @@ -1,4 +1,5 @@ -using BenchmarkDotNet.Attributes; +using System.Buffers; +using BenchmarkDotNet.Attributes; using System.Text; namespace DBCD.Benchmark.Benchmarks @@ -52,38 +53,47 @@ public static Dictionary ReadStringTable(this BinaryReader reader, { if (stringTableSize == 0) return []; - + var stringTable = new Dictionary(stringTableSize / 0x20); - Span stringTableBytes = stackalloc byte[stringTableSize]; - _ = reader.Read(stringTableBytes); - - int start = 0; - for (int i = 0; i < stringTableBytes.Length; ++i) + byte[] stringTableBytes = + ArrayPool.Shared.Rent(stringTableSize); // may return a lager buffer than requested + Span bufferSpan = stringTableBytes.AsSpan(0, stringTableSize); + _ = reader.Read(bufferSpan); + + try { - if (stringTableBytes[i] == 0) + int start = 0; + for (int i = 0; i < stringTableBytes.Length; ++i) + { + if (stringTableBytes[i] == 0) + { + string str = Encoding.UTF8.GetString(bufferSpan.Slice(start, i - start)); + if (usePos) + stringTable[reader.BaseStream.Position - stringTableSize + start] = str; + else + stringTable[baseOffset + start] = str; + + start = i + 1; + } + } + + // Trailing string + if (start < stringTableBytes.Length) { - string str = Encoding.UTF8.GetString(stringTableBytes.Slice(start, i - start)); + string str = Encoding.UTF8.GetString(bufferSpan.Slice(start)); if (usePos) stringTable[reader.BaseStream.Position - stringTableSize + start] = str; else stringTable[baseOffset + start] = str; - - start = i + 1; } + + return stringTable; } - - // Trailing string - if (start < stringTableBytes.Length) + finally { - string str = Encoding.UTF8.GetString(stringTableBytes.Slice(start)); - if (usePos) - stringTable[reader.BaseStream.Position - stringTableSize + start] = str; - else - stringTable[baseOffset + start] = str; + ArrayPool.Shared.Return(stringTableBytes); } - - return stringTable; } } } diff --git a/DBCD.IO/Extensions.cs b/DBCD.IO/Extensions.cs index b72c0b8..8c3d7a1 100644 --- a/DBCD.IO/Extensions.cs +++ b/DBCD.IO/Extensions.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Collections.Generic; using System.Diagnostics.Contracts; using System.IO; @@ -70,35 +71,43 @@ public static Dictionary ReadStringTable(this BinaryReader reader, var stringTable = new Dictionary(stringTableSize / 0x20); - Span stringTableBytes = stackalloc byte[stringTableSize]; - _ = reader.Read(stringTableBytes); - - int start = 0; - for (int i = 0; i < stringTableBytes.Length; ++i) + byte[] stringTableBytes = ArrayPool.Shared.Rent(stringTableSize); // may return a lager buffer than requested + Span bufferSpan = stringTableBytes.AsSpan(0, stringTableSize); + _ = reader.Read(bufferSpan); + + try { - if (stringTableBytes[i] == 0) + int start = 0; + for (int i = 0; i < stringTableBytes.Length; ++i) + { + if (stringTableBytes[i] == 0) + { + string str = Encoding.UTF8.GetString(bufferSpan.Slice(start, i - start)); + if (usePos) + stringTable[reader.BaseStream.Position - stringTableSize + start] = str; + else + stringTable[baseOffset + start] = str; + + start = i + 1; + } + } + + // Trailing string + if (start < stringTableBytes.Length) { - string str = Encoding.UTF8.GetString(stringTableBytes.Slice(start, i - start)); + string str = Encoding.UTF8.GetString(bufferSpan.Slice(start)); if (usePos) stringTable[reader.BaseStream.Position - stringTableSize + start] = str; else stringTable[baseOffset + start] = str; - - start = i + 1; } + + return stringTable; } - - // Trailing string - if (start < stringTableBytes.Length) + finally { - string str = Encoding.UTF8.GetString(stringTableBytes.Slice(start)); - if (usePos) - stringTable[reader.BaseStream.Position - stringTableSize + start] = str; - else - stringTable[baseOffset + start] = str; + ArrayPool.Shared.Return(stringTableBytes); } - - return stringTable; } public static T[] ReadArray(this BinaryReader reader) where T : struct From dc3490f59b7c35ccb2b935ca8b661e72cc658f87 Mon Sep 17 00:00:00 2001 From: Ovahlord <18347559+Ovahlord@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:52:03 +0200 Subject: [PATCH 3/4] removed some stupid auto-formatting --- DBCD.Benchmark/Benchmarks/StringTableBench.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DBCD.Benchmark/Benchmarks/StringTableBench.cs b/DBCD.Benchmark/Benchmarks/StringTableBench.cs index fca7c9c..9626f3b 100644 --- a/DBCD.Benchmark/Benchmarks/StringTableBench.cs +++ b/DBCD.Benchmark/Benchmarks/StringTableBench.cs @@ -56,8 +56,7 @@ public static Dictionary ReadStringTable(this BinaryReader reader, var stringTable = new Dictionary(stringTableSize / 0x20); - byte[] stringTableBytes = - ArrayPool.Shared.Rent(stringTableSize); // may return a lager buffer than requested + byte[] stringTableBytes = ArrayPool.Shared.Rent(stringTableSize); // may return a lager buffer than requested Span bufferSpan = stringTableBytes.AsSpan(0, stringTableSize); _ = reader.Read(bufferSpan); From 33106bf1aaefd52c07c229f1c2939d8e59a59fab Mon Sep 17 00:00:00 2001 From: Ovahlord <18347559+Ovahlord@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:55:43 +0200 Subject: [PATCH 4/4] Fixed a possible out of range exception caused by using the rented buffer's size instead of the actual size and use buffer pooling for array loading as well --- DBCD.Benchmark/Benchmarks/StringTableBench.cs | 4 +-- DBCD.IO/Extensions.cs | 36 +++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/DBCD.Benchmark/Benchmarks/StringTableBench.cs b/DBCD.Benchmark/Benchmarks/StringTableBench.cs index 9626f3b..ffaa1be 100644 --- a/DBCD.Benchmark/Benchmarks/StringTableBench.cs +++ b/DBCD.Benchmark/Benchmarks/StringTableBench.cs @@ -63,7 +63,7 @@ public static Dictionary ReadStringTable(this BinaryReader reader, try { int start = 0; - for (int i = 0; i < stringTableBytes.Length; ++i) + for (int i = 0; i < bufferSpan.Length; ++i) { if (stringTableBytes[i] == 0) { @@ -78,7 +78,7 @@ public static Dictionary ReadStringTable(this BinaryReader reader, } // Trailing string - if (start < stringTableBytes.Length) + if (start < bufferSpan.Length) { string str = Encoding.UTF8.GetString(bufferSpan.Slice(start)); if (usePos) diff --git a/DBCD.IO/Extensions.cs b/DBCD.IO/Extensions.cs index 8c3d7a1..b7661ed 100644 --- a/DBCD.IO/Extensions.cs +++ b/DBCD.IO/Extensions.cs @@ -68,7 +68,7 @@ public static Dictionary ReadStringTable(this BinaryReader reader, { if (stringTableSize == 0) return []; - + var stringTable = new Dictionary(stringTableSize / 0x20); byte[] stringTableBytes = ArrayPool.Shared.Rent(stringTableSize); // may return a lager buffer than requested @@ -78,7 +78,7 @@ public static Dictionary ReadStringTable(this BinaryReader reader, try { int start = 0; - for (int i = 0; i < stringTableBytes.Length; ++i) + for (int i = 0; i < bufferSpan.Length; ++i) { if (stringTableBytes[i] == 0) { @@ -93,7 +93,7 @@ public static Dictionary ReadStringTable(this BinaryReader reader, } // Trailing string - if (start < stringTableBytes.Length) + if (start < bufferSpan.Length) { string str = Encoding.UTF8.GetString(bufferSpan.Slice(start)); if (usePos) @@ -113,19 +113,35 @@ public static Dictionary ReadStringTable(this BinaryReader reader, public static T[] ReadArray(this BinaryReader reader) where T : struct { int numBytes = (int)reader.ReadInt64(); - Span result = stackalloc byte[numBytes]; - _ = reader.Read(result); + byte[] buffer = ArrayPool.Shared.Rent(numBytes); // may return a lager buffer than requested + Span result = buffer.AsSpan(0, numBytes); - reader.BaseStream.Position += (0 - numBytes) & 0x07; - return MemoryMarshal.Cast(result).ToArray(); + try + { + _ = reader.Read(result); + reader.BaseStream.Position += (0 - numBytes) & 0x07; + return MemoryMarshal.Cast(result).ToArray(); + } + finally + { + ArrayPool.Shared.Return(buffer); + } } public static T[] ReadArray(this BinaryReader reader, int size) where T : struct { int numBytes = Unsafe.SizeOf() * size; - Span result = stackalloc byte[numBytes]; - _ = reader.Read(result); - return MemoryMarshal.Cast(result).ToArray(); + byte[] buffer = ArrayPool.Shared.Rent(numBytes); // may return a lager buffer than requested + Span result = buffer.AsSpan(0, numBytes); + try + { + _ = reader.Read(result); + return MemoryMarshal.Cast(result).ToArray(); + } + finally + { + ArrayPool.Shared.Return(buffer); + } } public static unsafe T[] CopyTo(this byte[] src) where T : struct