Skip to content

Reduce the amount of unnecessary heap allocations while parsing string tables and fields #34

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Ovahlord wants to merge 4 commits into
base: master
Choose a base branch
from
+104 −43
Open

Reduce the amount of unnecessary heap allocations while parsing string tables and fields #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b69c013
Reduce the amount of unnecessary heap allocations while parsing strin…
Ovahlord Jul 31, 2026
745e262
Refactor string table loading to use the array pool instead of stacka…
Ovahlord Aug 1, 2026
dc3490f
removed some stupid auto-formatting
Ovahlord Aug 1, 2026
33106bf
Fixed a possible out of range exception caused by using the rented bu…
Ovahlord Aug 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 38 additions & 16 deletions DBCD.Benchmark/Benchmarks/StringTableBench.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using BenchmarkDotNet.Attributes;
using System.Buffers;
using BenchmarkDotNet.Attributes;
using System.Text;

namespace DBCD.Benchmark.Benchmarks
Expand Down Expand Up @@ -50,27 +51,48 @@ public static string ReadCString(this BinaryReader reader)

public static Dictionary<long, string> ReadStringTable(this BinaryReader reader, int stringTableSize, int baseOffset = 0, bool usePos = false)
{
var StringTable = new Dictionary<long, string>(stringTableSize / 0x20);

if (stringTableSize == 0)
return StringTable;
return [];

var stringTable = new Dictionary<long, string>(stringTableSize / 0x20);

byte[] stringTableBytes = ArrayPool<byte>.Shared.Rent(stringTableSize); // may return a lager buffer than requested
Span<byte> bufferSpan = stringTableBytes.AsSpan(0, stringTableSize);
_ = reader.Read(bufferSpan);

var curOfs = 0;
var decoded = Encoding.UTF8.GetString(reader.ReadBytes(stringTableSize));
foreach (var str in decoded.Split('\0'))
try
{
if (curOfs == stringTableSize)
break;
int start = 0;
for (int i = 0; i < bufferSpan.Length; ++i)
{
if (stringTableBytes[i] == 0)
{
string str = Encoding.UTF8.GetString(bufferSpan.Slice(start, i - start));
if (usePos)
stringTable[reader.BaseStream.Position - stringTableSize + start] = str;
else
stringTable[baseOffset + start] = str;

if (usePos)
StringTable[(reader.BaseStream.Position - stringTableSize) + curOfs] = str;
else
StringTable[baseOffset + curOfs] = str;
start = i + 1;
}
}

curOfs += Encoding.UTF8.GetByteCount(str) + 1;
}
// Trailing string
if (start < bufferSpan.Length)
{
string str = Encoding.UTF8.GetString(bufferSpan.Slice(start));
if (usePos)
stringTable[reader.BaseStream.Position - stringTableSize + start] = str;
else
stringTable[baseOffset + start] = str;
}

return StringTable;
return stringTable;
}
finally
{
ArrayPool<byte>.Shared.Return(stringTableBytes);
}
}
}
}
93 changes: 66 additions & 27 deletions DBCD.IO/Extensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;

namespace DBCD.IO
Expand Down Expand Up @@ -50,11 +52,11 @@ public static FieldCache<T>[] ToFieldCache<T>(this Type type)

public static T Read<T>(this BinaryReader reader) where T : struct
{
byte[] result = reader.ReadBytes(Unsafe.SizeOf<T>());
Span<byte> result = stackalloc byte[Unsafe.SizeOf<T>()];
_ = reader.Read(result);
return Unsafe.ReadUnaligned<T>(ref result[0]);
}


/// <summary>
/// Reads a NUL-separated string table from the current stream
/// </summary>
Expand All @@ -64,45 +66,82 @@ public static T Read<T>(this BinaryReader reader) where T : struct
/// <param name="baseOffset">Base offset to use for the string table keys</param>
public static Dictionary<long, string> ReadStringTable(this BinaryReader reader, int stringTableSize, int baseOffset = 0, bool usePos = false)
{
var StringTable = new Dictionary<long, string>(stringTableSize / 0x20);

if(stringTableSize == 0)
return StringTable;
if (stringTableSize == 0)
return [];

var curOfs = 0;
var decoded = Encoding.UTF8.GetString(reader.ReadBytes(stringTableSize));
foreach (var str in decoded.Split('\0'))
{
if (curOfs == stringTableSize)
break;
var stringTable = new Dictionary<long, string>(stringTableSize / 0x20);

if(usePos)
StringTable[(reader.BaseStream.Position - stringTableSize) + curOfs] = str;
else
StringTable[baseOffset + curOfs] = str;
byte[] stringTableBytes = ArrayPool<byte>.Shared.Rent(stringTableSize); // may return a lager buffer than requested
Span<byte> bufferSpan = stringTableBytes.AsSpan(0, stringTableSize);
_ = reader.Read(bufferSpan);

curOfs += Encoding.UTF8.GetByteCount(str) + 1;
try
{
int start = 0;
for (int i = 0; i < bufferSpan.Length; ++i)
{
if (stringTableBytes[i] == 0)
{
string str = Encoding.UTF8.GetString(bufferSpan.Slice(start, i - start));
if (usePos)
stringTable[reader.BaseStream.Position - stringTableSize + start] = str;
else
stringTable[baseOffset + start] = str;

start = i + 1;
}
}

// Trailing string
if (start < bufferSpan.Length)
{
string str = Encoding.UTF8.GetString(bufferSpan.Slice(start));
if (usePos)
stringTable[reader.BaseStream.Position - stringTableSize + start] = str;
else
stringTable[baseOffset + start] = str;
}

return stringTable;
}
finally
{
ArrayPool<byte>.Shared.Return(stringTableBytes);
}

return StringTable;
}

public static T[] ReadArray<T>(this BinaryReader reader) where T : struct
{
int numBytes = (int)reader.ReadInt64();
byte[] buffer = ArrayPool<byte>.Shared.Rent(numBytes); // may return a lager buffer than requested
Span<byte> result = buffer.AsSpan(0, numBytes);

byte[] result = reader.ReadBytes(numBytes);

reader.BaseStream.Position += (0 - numBytes) & 0x07;
return result.CopyTo<T>();
try
{
_ = reader.Read(result);
reader.BaseStream.Position += (0 - numBytes) & 0x07;
return MemoryMarshal.Cast<byte, T>(result).ToArray();
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
}

public static T[] ReadArray<T>(this BinaryReader reader, int size) where T : struct
{
int numBytes = Unsafe.SizeOf<T>() * size;

byte[] result = reader.ReadBytes(numBytes);
return result.CopyTo<T>();
byte[] buffer = ArrayPool<byte>.Shared.Rent(numBytes); // may return a lager buffer than requested
Span<byte> result = buffer.AsSpan(0, numBytes);
try
{
_ = reader.Read(result);
return MemoryMarshal.Cast<byte, T>(result).ToArray();
}
finally
{
ArrayPool<byte>.Shared.Return(buffer);
}
}

public static unsafe T[] CopyTo<T>(this byte[] src) where T : struct
Expand Down Expand Up @@ -171,7 +210,7 @@ public static string ReadCString(this BinaryReader reader)
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public static string ReadCString(this BinaryReader reader, Encoding encoding)
{
var bytes = new System.Collections.Generic.List<byte>(0x20);
var bytes = new List<byte>(0x20);
byte b;
while ((b = reader.ReadByte()) != 0)
bytes.Add(b);
Expand Down