From c1c8d2099b19da606d10da11457c071d0b1315d1 Mon Sep 17 00:00:00 2001 From: NoobInCoding Date: Sat, 18 Apr 2026 19:53:47 +0330 Subject: [PATCH 1/8] Replace CUE4Parse NuGet package with a git submodule Rationale: we need source-level access to CUE4Parse to debug API quirks, apply local patches, and pick up upstream fixes without waiting for a package release. Changes: - UnrealReZen.csproj: drop PackageReference "CUE4Parse" 4.28.0; add ProjectReference to ..\external\CUE4Parse\CUE4Parse\CUE4Parse.csproj - .gitmodules: pin external/CUE4Parse to FabianFG/CUE4Parse - Program.cs / Compressions.cs: adapt to the current CUE4Parse API (OodleHelper.OodleFileName constant, ref-path DownloadOodleDll signature) --- .gitmodules | 3 +++ UnrealReZen/Core/Compression/Compressions.cs | 2 +- UnrealReZen/Program.cs | 5 +++-- UnrealReZen/UnrealReZen.csproj | 5 ++++- external/CUE4Parse | 1 + 5 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 .gitmodules create mode 160000 external/CUE4Parse diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a898a2f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/CUE4Parse"] + path = external/CUE4Parse + url = https://github.com/FabianFG/CUE4Parse.git diff --git a/UnrealReZen/Core/Compression/Compressions.cs b/UnrealReZen/Core/Compression/Compressions.cs index d5789ce..fa5700f 100644 --- a/UnrealReZen/Core/Compression/Compressions.cs +++ b/UnrealReZen/Core/Compression/Compressions.cs @@ -55,7 +55,7 @@ private static byte[] CompressZlib(byte[] inData) private static byte[] CompressOodle(byte[] inData) { const OodleCompressor compressor = OodleCompressor.Kraken; - using var _oodle = new Oodle(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.OodleHelper.OODLE_DLL_NAME)); + using var _oodle = new Oodle(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.OodleHelper.OodleFileName)); var compressedBufferSize = (int)_oodle.GetCompressedBufferSizeNeeded(compressor, inData.Length); var compressedBuffer = new byte[compressedBufferSize]; var compressedSize = (int)_oodle.Compress(compressor, OodleCompressionLevel.Max, inData, compressedBuffer); diff --git a/UnrealReZen/Program.cs b/UnrealReZen/Program.cs index 331eb18..0b54bb0 100644 --- a/UnrealReZen/Program.cs +++ b/UnrealReZen/Program.cs @@ -71,9 +71,10 @@ static void Main(string[] args) static void RunOptionsAndReturnExitCode(Options opts) { Constants.ToolDirectory = AppDomain.CurrentDomain.BaseDirectory; - if (!OodleHelper.DownloadOodleDll(Path.Combine(Constants.ToolDirectory, OodleHelper.OODLE_DLL_NAME))) + string? oodlePath = Path.Combine(Constants.ToolDirectory, OodleHelper.OodleFileName); + if (!OodleHelper.DownloadOodleDll(ref oodlePath)) { - Log.Fatal("UnrealReZen failed to download the oodle dll. please check you internet connection or place oo2core_9_win64.dll in the tool directory"); + Log.Fatal($"UnrealReZen failed to download the oodle dll. please check you internet connection or place {OodleHelper.OodleFileName} in the tool directory"); Console.ReadLine(); return; } diff --git a/UnrealReZen/UnrealReZen.csproj b/UnrealReZen/UnrealReZen.csproj index a11087b..21ea3b4 100644 --- a/UnrealReZen/UnrealReZen.csproj +++ b/UnrealReZen/UnrealReZen.csproj @@ -11,10 +11,13 @@ - + + + + True diff --git a/external/CUE4Parse b/external/CUE4Parse new file mode 160000 index 0000000..9539d4d --- /dev/null +++ b/external/CUE4Parse @@ -0,0 +1 @@ +Subproject commit 9539d4d86aef568f27957796249891d125d2c223 From ffcab3a454b9e7edd23bacd4c3fe2ecf2216ecd9 Mon Sep 17 00:00:00 2001 From: NoobInCoding Date: Sat, 18 Apr 2026 19:54:29 +0330 Subject: [PATCH 2/8] Fix packing bugs: divide-by-zero, broken MMF, OOM on AES, Windows-only paths WriteProgressBar no longer divides by zero on single-file packs - Clamp denominator with Math.Max(total, 1) so passing one file to pack doesn't throw DivideByZeroException. Replace the broken MemoryMappedHelpers pattern with ChunkSource - CreateMemoryMappedFileFromByteArray used CreateNew inside a using block then OpenExisting after dispose, so the mapping was already released when callers tried to read it. Deleted MemoryMappedHelpers.cs. - New ChunkSource abstraction (FromFile / FromBytes, ReadInto, Length) gives the packer a single type that works for both on-disk inputs (memory-mapped) and the in-memory dependencies blob. Stream AES encryption instead of loading the entire .ucas into memory - Old path: File.ReadAllBytes -> EncryptAES(byte[]) -> File.WriteAllBytes, which OOM'd on large packs (>2 GB). - New path: CryptoStream from src .ucas to a temp file, then File.Move replaces the original. Cross-platform path handling - Path.GetRelativePath + Replace('\','/') instead of Replace("\\","") string math that only happened to work on Windows. Type safety - Replace `dynamic b = dep` with provider.Files.OfType(); avoids runtime DLR dispatch and the NRE it throws on unexpected types. - Tighten nullable annotations on DirIndexWrapper helpers (dentry / lastDentry now correctly typed FIoDirectoryIndexEntry?). CUE4Parse API migration - Oodle: OodleHelper.OODLE_DLL_NAME -> OodleHelper.OodleFileName, DownloadOodleDll(string) -> DownloadOodleDll(ref string? path). --- UnrealReZen/Core/FIoPack.cs | 175 +++++++++--------- UnrealReZen/Core/FIoStructs.cs | 42 ++--- UnrealReZen/Core/Helpers/ChunkSource.cs | 76 ++++++++ .../Core/Helpers/MemoryMappedHelpers.cs | 51 ----- UnrealReZen/Program.cs | 34 ++-- 5 files changed, 204 insertions(+), 174 deletions(-) create mode 100644 UnrealReZen/Core/Helpers/ChunkSource.cs delete mode 100644 UnrealReZen/Core/Helpers/MemoryMappedHelpers.cs diff --git a/UnrealReZen/Core/FIoPack.cs b/UnrealReZen/Core/FIoPack.cs index a318d75..0f2c232 100644 --- a/UnrealReZen/Core/FIoPack.cs +++ b/UnrealReZen/Core/FIoPack.cs @@ -3,7 +3,7 @@ using CUE4Parse.UE4.Objects.Core.Misc; using CUE4Parse.UE4.Versions; using System.Globalization; -using System.IO.MemoryMappedFiles; +using System.Security.Cryptography; using System.Text; using UnrealReZen.Core.Compression; using UnrealReZen.Core.Helpers; @@ -47,11 +47,9 @@ public static int PackToCasToc(string dir, Dependency m, string outFilename, str fdata.PackFilesToUcas(m, dir, outFilename, compression, depver); - if (aes.KeyString != Constants.DefaultAES) + if (!string.Equals(aes.KeyString, Constants.DefaultAES, StringComparison.OrdinalIgnoreCase)) { - var b = File.ReadAllBytes(Path.ChangeExtension(outFilename, ".ucas")); - var encrypted = CryptographyHelpers.EncryptAES(b, aes.Key); - File.WriteAllBytes(Path.ChangeExtension(outFilename, ".ucas"), encrypted); + EncryptUcasInPlace(Path.ChangeExtension(outFilename, ".ucas"), aes.Key); } var utocBytes = fdata.ConstructUtocFile(compression, aes, gameVer); @@ -79,32 +77,20 @@ public static void PackFilesToUcas(this List files, Dependency m, m.Deps.ChunkIDToDependencies = subsetDependencies; - var compMethodNumber = !compression.Equals("none", StringComparison.CurrentCultureIgnoreCase) ? (byte)1 : (byte)0; + var compMethodNumber = !compression.Equals("none", StringComparison.OrdinalIgnoreCase) ? (byte)1 : (byte)0; var compFun = CompressionUtils.GetCompressionFunction(compression) ?? throw new Exception("Could not find " + compression + " method. Please use None, Oodle or Zlib"); using var f = File.Open(Path.ChangeExtension(outFilename, ".ucas"), FileMode.Create); + var readBuffer = new byte[Constants.CompSize]; for (int i = 0; i < files.Count; i++) { - WriteProgressBar(i, files.Count - 1); - - MemoryMappedFile mmf; - long SizeOfmmf; - string pathToread = Path.Combine(dir.Replace("/", "\\"), files[i].FilePath.Replace("/", "\\")); - if (!File.Exists(pathToread)) - { - if (files[i].FilePath != Constants.DepFileName) throw new Exception("File doesn't exist, and also its not the dependency file."); - byte[] ManifestCreatedFile = depver == FIoDependencyFormat.UE4 ? m.WriteDependenciesAsUE4() : m.WriteDependenciesAsUE5(); - mmf = MemoryMappedHelpers.CreateMemoryMappedFileFromByteArray(ManifestCreatedFile, files[i].FilePath); - SizeOfmmf = ManifestCreatedFile.LongLength; - files[i].FilePath = ""; - } - else - { - mmf = MemoryMappedFile.CreateFromFile(pathToread, FileMode.Open, Path.GetFileNameWithoutExtension(pathToread)); - SizeOfmmf = new FileInfo(pathToread).Length; - } + WriteProgressBar(i + 1, files.Count); + var pathToRead = Path.Combine(dir, files[i].FilePath); + using ChunkSource source = File.Exists(pathToRead) + ? ChunkSource.FromFile(pathToRead) + : CreateDependencyChunkSource(files[i], m, depver); - files[i].OffLen.SetLength((ulong)SizeOfmmf); + files[i].OffLen.SetLength((ulong)source.Length); if (i == 0) { @@ -117,82 +103,93 @@ public static void PackFilesToUcas(this List files, Dependency m, files[i].OffLen.SetOffset(off); } - files[i].Metadata.ChunkHash = new FIoChunkHash(mmf.SHA1Hash()); + files[i].Metadata.ChunkHash = new FIoChunkHash(source.ComputeSha1()); files[i].Metadata.Flags = FIoStoreTocEntryMetaFlags.CompressedMetaFlag; - long PosOfReaded = 0; - long RemainSize = SizeOfmmf; - while (PosOfReaded != SizeOfmmf) + long readPos = 0; + while (readPos < source.Length) { - var block = new FIoStoreTocCompressedBlockEntry(); - var chunkLen = RemainSize; - if (chunkLen > Constants.CompSize) - { - chunkLen = Constants.CompSize; - } - RemainSize -= chunkLen; - var chunk = mmf.ReadBytesOfFile(PosOfReaded, chunkLen); - PosOfReaded += chunkLen; - var cChunkPtr = compFun(chunk); - var compressedChunk = cChunkPtr.ToArray(); + int chunkLen = (int)Math.Min(Constants.CompSize, source.Length - readPos); + source.ReadInto(readPos, readBuffer, 0, chunkLen); + readPos += chunkLen; + + byte[] compressedChunk = compFun(chunkLen == readBuffer.Length ? readBuffer : readBuffer[..chunkLen]); - block.CompressionMethod = compMethodNumber; + var block = new FIoStoreTocCompressedBlockEntry + { + CompressionMethod = compMethodNumber + }; block.SetOffset((ulong)f.Position); block.SetUncompressedSize((uint)chunkLen); block.SetCompressedSize((uint)compressedChunk.Length); - - compressedChunk = [.. compressedChunk, .. CryptographyHelpers.GetRandomBytes(0x10 - compressedChunk.Length % 0x10 & 0x10 - 1)]; files[i].CompressionBlocks.Add(block); f.Write(compressedChunk, 0, compressedChunk.Length); + int padLen = (0x10 - compressedChunk.Length % 0x10) & 0x0F; + if (padLen > 0) + { + Span pad = stackalloc byte[16]; + RandomNumberGenerator.Fill(pad[..padLen]); + f.Write(pad[..padLen]); + } } - mmf.Dispose(); } - // Add a line feed for the progress bar Console.WriteLine(""); } - public static void WriteProgressBar(int count, int maxCount) + private static ChunkSource CreateDependencyChunkSource(AssetMetadata entry, Dependency m, FIoDependencyFormat depver) + { + if (entry.FilePath != Constants.DepFileName) + { + throw new FileNotFoundException($"Content file not found: {entry.FilePath}"); + } + byte[] depBytes = depver == FIoDependencyFormat.UE4 ? m.WriteDependenciesAsUE4() : m.WriteDependenciesAsUE5(); + entry.FilePath = ""; + return ChunkSource.FromBytes(depBytes); + } + + private static void EncryptUcasInPlace(string path, byte[] aesKey) + { + string tempPath = path + ".enc.tmp"; + using (var aes = System.Security.Cryptography.Aes.Create()) + { + aes.Key = aesKey; + aes.Mode = CipherMode.ECB; + aes.Padding = PaddingMode.None; + using var input = File.Open(path, FileMode.Open, FileAccess.Read); + using var output = File.Open(tempPath, FileMode.Create, FileAccess.Write); + using var encryptor = aes.CreateEncryptor(); + using var crypto = new CryptoStream(output, encryptor, CryptoStreamMode.Write); + input.CopyTo(crypto); + } + File.Move(tempPath, path, overwrite: true); + } + + public static void WriteProgressBar(int count, int total) { - // Display a progress bar on the console. - // e.g. WriteProgressBar(54, 100) - // [##########..........] 54/100 const int MaxProgress = 20; - var progress = count * MaxProgress / maxCount; + int denom = Math.Max(total, 1); + int progress = Math.Min(count * MaxProgress / denom, MaxProgress); string str = new string('#', progress) + new string('.', MaxProgress - progress); - Console.Write($"\r[{str}] {count}/{maxCount}"); + Console.Write($"\r[{str}] {count}/{total}"); } public static byte[] DeparseDirectoryIndex(List files) { - var wrapper = new DirIndexWrapper(); - var dirIndexEntries = new List(); - var fileIndexEntries = new List(); - - var strmap = new Dictionary(); - foreach (var v in files) + var strIdx = new Dictionary(); + var strSlice = new List(); + foreach (var file in files) { - var dirfiles = v.FilePath.Split('/'); - if (dirfiles[0] == "") - { - dirfiles = dirfiles.Skip(1).ToArray(); - } - - foreach (var str in dirfiles) + foreach (var segment in SplitPath(file.FilePath)) { - strmap[str] = true; + if (strIdx.TryAdd(segment, strSlice.Count)) + { + strSlice.Add(segment); + } } } - var strSlice = strmap.Keys.ToList(); - var strIdx = new Dictionary(); - for (int iv = 0; iv < strSlice.Count; iv++) - { - strIdx.Add(strSlice[iv], iv); - } - - var root = new FIoDirectoryIndexEntry { Name = Constants.NoneEntry, @@ -201,26 +198,34 @@ public static byte[] DeparseDirectoryIndex(List files) FirstFileEntry = Constants.NoneEntry, }; - dirIndexEntries.Add(root); - wrapper.Dirs = dirIndexEntries; - wrapper.Files = fileIndexEntries; - wrapper.StrTable = strIdx; - wrapper.StrSlice = [.. strSlice]; + var wrapper = new DirIndexWrapper( + new List { root }, + new List(files.Count), + strIdx, + strSlice.ToArray()); for (var i = 0; i < files.Count; i++) { - var fpathSections = files[i].FilePath.Split('/'); - if (fpathSections[0] == "") - { - fpathSections = fpathSections.Skip(1).ToArray(); - } - - root.AddFile(fpathSections, (uint)i, wrapper); + root.AddFile(SplitPath(files[i].FilePath).ToArray(), (uint)i, wrapper); } return wrapper.ToBytes(); } + private static IEnumerable SplitPath(string path) + { + int start = path.StartsWith('/') ? 1 : 0; + for (int i = start; i < path.Length; i++) + { + if (path[i] == '/') + { + if (i > start) yield return path[start..i]; + start = i + 1; + } + } + if (start < path.Length) yield return path[start..]; + } + public static byte[] ConstructUtocFile(this List files, string compression, FAesKey AESKey, EGame gameVer) { var udata = new UToc(new UTocHeader(), [], "", []); diff --git a/UnrealReZen/Core/FIoStructs.cs b/UnrealReZen/Core/FIoStructs.cs index a20df74..fa43835 100644 --- a/UnrealReZen/Core/FIoStructs.cs +++ b/UnrealReZen/Core/FIoStructs.cs @@ -92,8 +92,8 @@ public void AddFile(string[] fpathSections, uint fIndex, DirIndexWrapper structu } else { - FIoDirectoryIndexEntry dentry = structure.Dirs[(int)FirstChildEntry]; - FIoDirectoryIndexEntry lastDentry = null; + FIoDirectoryIndexEntry? dentry = structure.Dirs[(int)FirstChildEntry]; + FIoDirectoryIndexEntry? lastDentry = null; while (dentry != null && dentry.Name != currDirNameIndex) { @@ -200,10 +200,10 @@ public static FIoChunkID FromHexString(string hexString) public class DirIndexWrapper { - public List Dirs { get; set; } - public List Files { get; set; } - public Dictionary StrTable { get; set; } - public string[] StrSlice { get; set; } + public List Dirs { get; } + public List Files { get; } + public Dictionary StrTable { get; } + public string[] StrSlice { get; } public DirIndexWrapper(List dirs, List files, Dictionary strTable, string[] strSlice) { @@ -212,43 +212,33 @@ public DirIndexWrapper(List dirs, List()); + } + var mmf = MemoryMappedFile.CreateFromFile(path, FileMode.Open, null, 0, MemoryMappedFileAccess.Read); + var accessor = mmf.CreateViewAccessor(0, length, MemoryMappedFileAccess.Read); + return new ChunkSource(mmf, accessor, length); + } + + public static ChunkSource FromBytes(byte[] bytes) => new(bytes); + + public void ReadInto(long offset, byte[] dest, int destOffset, int count) + { + if (_accessor != null) + { + _accessor.ReadArray(offset, dest, destOffset, count); + } + else + { + Array.Copy(_bytes!, offset, dest, destOffset, count); + } + } + + public byte[] ComputeSha1() + { + using var sha1 = SHA1.Create(); + const int bufferSize = 64 * 1024; + var buffer = new byte[bufferSize]; + long position = 0; + while (position < Length) + { + int bytesToRead = (int)Math.Min(bufferSize, Length - position); + ReadInto(position, buffer, 0, bytesToRead); + sha1.TransformBlock(buffer, 0, bytesToRead, buffer, 0); + position += bytesToRead; + } + sha1.TransformFinalBlock(buffer, 0, 0); + return sha1.Hash!; + } + + public void Dispose() + { + _accessor?.Dispose(); + _mmf?.Dispose(); + } + } +} diff --git a/UnrealReZen/Core/Helpers/MemoryMappedHelpers.cs b/UnrealReZen/Core/Helpers/MemoryMappedHelpers.cs deleted file mode 100644 index d6b0fa4..0000000 --- a/UnrealReZen/Core/Helpers/MemoryMappedHelpers.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System.IO.MemoryMappedFiles; -using System.Security.Cryptography; - -namespace UnrealReZen.Core.Helpers -{ - public static class MemoryMappedHelpers - { - public static byte[] ReadBytesOfFile(this MemoryMappedFile mmf, long offset, long length) - { - using (MemoryMappedViewAccessor accessor = mmf.CreateViewAccessor(offset, length)) - { - byte[] data = new byte[length]; - accessor.ReadArray(0, data, 0, data.Length); - return data; - } - } - public static MemoryMappedFile CreateMemoryMappedFileFromByteArray(byte[] data, string filePath) - { - using (MemoryMappedFile mmf = MemoryMappedFile.CreateNew(Path.GetFileNameWithoutExtension(filePath), data.Length)) - { - using (MemoryMappedViewAccessor accessor = mmf.CreateViewAccessor()) - { - accessor.WriteArray(0, data, 0, data.Length); - } - return MemoryMappedFile.OpenExisting(Path.GetFileNameWithoutExtension(filePath)); - } - } - public static byte[] SHA1Hash(this MemoryMappedFile mmf) - { - using MemoryMappedViewAccessor accessor = mmf.CreateViewAccessor(); - long length = accessor.Capacity; - int bufferSize = 4096; - byte[] buffer = new byte[bufferSize]; - - using SHA1 sha1 = SHA1.Create(); - long position = 0; - while (position < length) - { - int bytesToRead = (int)Math.Min(bufferSize, length - position); - - accessor.ReadArray(position, buffer, 0, bytesToRead); - sha1.TransformBlock(buffer, 0, bytesToRead, buffer, 0); - - position += bytesToRead; - } - - sha1.TransformFinalBlock(buffer, 0, 0); - return sha1.Hash; - } - } -} diff --git a/UnrealReZen/Program.cs b/UnrealReZen/Program.cs index 0b54bb0..902bdff 100644 --- a/UnrealReZen/Program.cs +++ b/UnrealReZen/Program.cs @@ -151,26 +151,36 @@ static void RunOptionsAndReturnExitCode(Options opts) } foreach (var file in FilesToRepack) { - string filename = file.Replace(opts.ContentPath + "\\", "").Replace("\\", "/"); + string filename = Path.GetRelativePath(opts.ContentPath, file).Replace('\\', '/'); Log.Information("Mounting " + Path.GetFileName(filename)); - var filedata = provider.Files.Values.Where(a => Path.GetExtension(((AbstractVfsReader)((VfsEntry)a).Vfs).Name) == ".utoc" && a.Path.Equals(filename, StringComparison.CurrentCultureIgnoreCase)); - if (filedata == null || !filedata.Any()) + var matches = provider.Files.Values + .OfType() + .Where(a => Path.GetExtension(((AbstractVfsReader)a.Vfs).Name) == ".utoc" + && a.Path.Equals(filename, StringComparison.OrdinalIgnoreCase)) + .ToList(); + if (matches.Count == 0) { Log.Warning("Skipping " + filename + " because its not found in archives."); continue; } - foreach (var dep in filedata) + foreach (var entry in matches) { - dynamic b = dep; - FIoChunkId c = b.ChunkId; - m.Files.Add(new ManifestFile { Filepath = dep.Path, ChunkID = new FIoChunkID(c.ChunkId, 0, 0, c.ChunkType) }); - IoStoreReader IoFile = b.IoStoreReader; - if (IoFile.ContainerHeader != null) + FIoChunkId chunkId = entry.ChunkId; + m.Files.Add(new ManifestFile { - foreach (var st in IoFile.ContainerHeader.StoreEntries) + Filepath = entry.Path, + ChunkID = new FIoChunkID(chunkId.ChunkId, 0, 0, chunkId.ChunkType) + }); + + var header = entry.IoStoreReader.ContainerHeader; + if (header != null) + { + foreach (var storeEntry in header.StoreEntries) { - if (m.Deps.ChunkIDToDependencies.ContainsKey(c.ChunkId)) continue; - m.Deps.ChunkIDToDependencies.Add(c.ChunkId, st); + if (!m.Deps.ChunkIDToDependencies.ContainsKey(chunkId.ChunkId)) + { + m.Deps.ChunkIDToDependencies.Add(chunkId.ChunkId, storeEntry); + } } } } From 75836044a476c97a497707025d182de4f29dcd6a Mon Sep 17 00:00:00 2001 From: NoobInCoding Date: Sat, 18 Apr 2026 19:54:47 +0330 Subject: [PATCH 3/8] Performance: one-pass packing, cached compressors, O(1) entry lookup Cache Zlib / Oodle native instances in static Lazy - Compressions.cs previously constructed a fresh Zlibng or Oodle wrapper (which reloads the native DLL) for every 64 KB compression block. On a 1 GB pack that was ~16 000 DLL init cycles. - Now held in Lazy / Lazy with isThreadSafe=true so the DLL is loaded exactly once per process. Fold SHA1 into the compression loop (single read pass) - Old: mmf.SHA1Hash() read the whole file, then the compression loop read it a second time block-by-block. - New: one loop reads each block once, feeds it into sha1.TransformBlock(..), compresses, writes. TransformFinalBlock at the end produces the chunk hash. Halves the bytes read from MMF. Build a utoc entry lookup once instead of scanning per-input - Old BuildManifest walked provider.Files (~all entries across all containers) for every file in the repack list: O(n*m). - New BuildUtocEntryLookup produces a Dictionary> keyed by entry.Path, case-insensitive, then each repack input is an O(1) probe: O(n+m) overall. Minor - Hoist the 16-byte stackalloc pad buffer out of the inner file loop (silences CA2014, allocates once per pack instead of once per file). - Drop ChunkSource.ComputeSha1; hashing moved into the one-pass loop. --- UnrealReZen/Core/Compression/Compressions.cs | 70 ++++++++------------ UnrealReZen/Core/FIoPack.cs | 11 +-- UnrealReZen/Core/Helpers/ChunkSource.cs | 18 ----- UnrealReZen/Program.cs | 19 ++++-- 4 files changed, 49 insertions(+), 69 deletions(-) diff --git a/UnrealReZen/Core/Compression/Compressions.cs b/UnrealReZen/Core/Compression/Compressions.cs index fa5700f..2560a4e 100644 --- a/UnrealReZen/Core/Compression/Compressions.cs +++ b/UnrealReZen/Core/Compression/Compressions.cs @@ -4,9 +4,8 @@ namespace UnrealReZen.Core.Compression { - public class CompressionUtils + public static class CompressionUtils { - public static readonly Dictionary> CompressionMethods = new(StringComparer.OrdinalIgnoreCase) { { "None", CompressNone }, @@ -15,64 +14,53 @@ public class CompressionUtils { "Lz4", CompressLZ4 } }; - public static byte[]? Compress(string method, byte[] inputData) - { - if (CompressionMethods.TryGetValue(method, out var compressionFunction)) - { - return compressionFunction(inputData); - } - return null; - } - public static Func? GetCompressionFunction(string method) - { - if (CompressionMethods.TryGetValue(method, out var compressionFunction)) - { - return compressionFunction; - } - return null; - } + => CompressionMethods.TryGetValue(method, out var fn) ? fn : null; - private static byte[] CompressNone(byte[] inData) - { - return inData; - } + private static readonly Lazy _zlib = new(() => + new Zlibng(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.ZlibHelper.DllName)), + isThreadSafe: true); + + private static readonly Lazy _oodle = new(() => + new Oodle(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.OodleHelper.OodleFileName)), + isThreadSafe: true); + + private static byte[] CompressNone(byte[] inData) => inData; private static byte[] CompressZlib(byte[] inData) { - using var _zlib = new Zlibng(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.ZlibHelper.DLL_NAME)); - var compressedBufferSize = (int)_zlib.CompressBound(inData.Length); - var compressedBuffer = new byte[compressedBufferSize]; - var compressionResult = _zlib.Compress(compressedBuffer, inData, out int compressedSize); - if (compressionResult.CompareTo(ZlibngCompressionResult.Ok) != 0) + var zlib = _zlib.Value; + var bufferSize = (int)zlib.CompressBound(inData.Length); + var buffer = new byte[bufferSize]; + var result = zlib.Compress(buffer, inData, out int compressedSize); + if (result != ZlibngCompressionResult.Ok) { - throw new Exception($"Zlib compression failed with error code {compressionResult}"); + throw new InvalidOperationException($"Zlib compression failed with error code {result}"); } - return compressedBuffer.AsSpan(0, compressedSize).ToArray(); - + Array.Resize(ref buffer, compressedSize); + return buffer; } private static byte[] CompressOodle(byte[] inData) { const OodleCompressor compressor = OodleCompressor.Kraken; - using var _oodle = new Oodle(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.OodleHelper.OodleFileName)); - var compressedBufferSize = (int)_oodle.GetCompressedBufferSizeNeeded(compressor, inData.Length); - var compressedBuffer = new byte[compressedBufferSize]; - var compressedSize = (int)_oodle.Compress(compressor, OodleCompressionLevel.Max, inData, compressedBuffer); - return [.. compressedBuffer.Take(compressedSize)]; + var oodle = _oodle.Value; + var bufferSize = (int)oodle.GetCompressedBufferSizeNeeded(compressor, inData.Length); + var buffer = new byte[bufferSize]; + var compressedSize = (int)oodle.Compress(compressor, OodleCompressionLevel.Max, inData, buffer); + Array.Resize(ref buffer, compressedSize); + return buffer; } private static byte[] CompressLZ4(byte[] inData) { - using (var inputStream = new MemoryStream(inData)) - using (var outputStream = new MemoryStream()) + using var inputStream = new MemoryStream(inData); + using var outputStream = new MemoryStream(inData.Length); using (var lz4Stream = LZ4Stream.Encode(outputStream)) { inputStream.CopyTo(lz4Stream); - lz4Stream.Flush(); - return outputStream.ToArray(); } - + return outputStream.ToArray(); } } -} \ No newline at end of file +} diff --git a/UnrealReZen/Core/FIoPack.cs b/UnrealReZen/Core/FIoPack.cs index 0f2c232..fe37f93 100644 --- a/UnrealReZen/Core/FIoPack.cs +++ b/UnrealReZen/Core/FIoPack.cs @@ -81,6 +81,7 @@ public static void PackFilesToUcas(this List files, Dependency m, var compFun = CompressionUtils.GetCompressionFunction(compression) ?? throw new Exception("Could not find " + compression + " method. Please use None, Oodle or Zlib"); using var f = File.Open(Path.ChangeExtension(outFilename, ".ucas"), FileMode.Create); var readBuffer = new byte[Constants.CompSize]; + Span padBuffer = stackalloc byte[16]; for (int i = 0; i < files.Count; i++) { WriteProgressBar(i + 1, files.Count); @@ -103,15 +104,16 @@ public static void PackFilesToUcas(this List files, Dependency m, files[i].OffLen.SetOffset(off); } - files[i].Metadata.ChunkHash = new FIoChunkHash(source.ComputeSha1()); files[i].Metadata.Flags = FIoStoreTocEntryMetaFlags.CompressedMetaFlag; + using var sha1 = SHA1.Create(); long readPos = 0; while (readPos < source.Length) { int chunkLen = (int)Math.Min(Constants.CompSize, source.Length - readPos); source.ReadInto(readPos, readBuffer, 0, chunkLen); readPos += chunkLen; + sha1.TransformBlock(readBuffer, 0, chunkLen, readBuffer, 0); byte[] compressedChunk = compFun(chunkLen == readBuffer.Length ? readBuffer : readBuffer[..chunkLen]); @@ -128,11 +130,12 @@ public static void PackFilesToUcas(this List files, Dependency m, int padLen = (0x10 - compressedChunk.Length % 0x10) & 0x0F; if (padLen > 0) { - Span pad = stackalloc byte[16]; - RandomNumberGenerator.Fill(pad[..padLen]); - f.Write(pad[..padLen]); + RandomNumberGenerator.Fill(padBuffer[..padLen]); + f.Write(padBuffer[..padLen]); } } + sha1.TransformFinalBlock(readBuffer, 0, 0); + files[i].Metadata.ChunkHash = new FIoChunkHash(sha1.Hash!); } Console.WriteLine(""); diff --git a/UnrealReZen/Core/Helpers/ChunkSource.cs b/UnrealReZen/Core/Helpers/ChunkSource.cs index 4381f8b..b6a6c27 100644 --- a/UnrealReZen/Core/Helpers/ChunkSource.cs +++ b/UnrealReZen/Core/Helpers/ChunkSource.cs @@ -1,5 +1,4 @@ using System.IO.MemoryMappedFiles; -using System.Security.Cryptography; namespace UnrealReZen.Core.Helpers { @@ -50,23 +49,6 @@ public void ReadInto(long offset, byte[] dest, int destOffset, int count) } } - public byte[] ComputeSha1() - { - using var sha1 = SHA1.Create(); - const int bufferSize = 64 * 1024; - var buffer = new byte[bufferSize]; - long position = 0; - while (position < Length) - { - int bytesToRead = (int)Math.Min(bufferSize, Length - position); - ReadInto(position, buffer, 0, bytesToRead); - sha1.TransformBlock(buffer, 0, bytesToRead, buffer, 0); - position += bytesToRead; - } - sha1.TransformFinalBlock(buffer, 0, 0); - return sha1.Hash!; - } - public void Dispose() { _accessor?.Dispose(); diff --git a/UnrealReZen/Program.cs b/UnrealReZen/Program.cs index 902bdff..753bbe7 100644 --- a/UnrealReZen/Program.cs +++ b/UnrealReZen/Program.cs @@ -149,16 +149,23 @@ static void RunOptionsAndReturnExitCode(Options opts) Log.Fatal("No valid files found in the content path"); return; } + var utocEntryLookup = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var entry in provider.Files.Values.OfType()) + { + if (Path.GetExtension(((AbstractVfsReader)entry.Vfs).Name) != ".utoc") continue; + if (!utocEntryLookup.TryGetValue(entry.Path, out var list)) + { + list = new List(); + utocEntryLookup[entry.Path] = list; + } + list.Add(entry); + } + foreach (var file in FilesToRepack) { string filename = Path.GetRelativePath(opts.ContentPath, file).Replace('\\', '/'); Log.Information("Mounting " + Path.GetFileName(filename)); - var matches = provider.Files.Values - .OfType() - .Where(a => Path.GetExtension(((AbstractVfsReader)a.Vfs).Name) == ".utoc" - && a.Path.Equals(filename, StringComparison.OrdinalIgnoreCase)) - .ToList(); - if (matches.Count == 0) + if (!utocEntryLookup.TryGetValue(filename, out var matches)) { Log.Warning("Skipping " + filename + " because its not found in archives."); continue; From bf595c446f2e6ccc17ae62afa54d4bc770d54ba9 Mon Sep 17 00:00:00 2001 From: NoobInCoding Date: Sat, 18 Apr 2026 19:55:04 +0330 Subject: [PATCH 4/8] Restructure: rename files, split Program.cs, drop dead code Renames (content also cleaned up) - UnrealReZen/Core/FIoConstants.cs -> Constants.cs (class was already named Constants; file name now matches) - UnrealReZen/Core/FIoUToc.cs -> UToc.cs (kept only UTocHeader, AssetMetadata, FIoContainerID; see below) Dead code removed - UToc wrapper class: never read back by anything; the packer only writes the UTOC bytes, so the reader-style wrapper was unused. - FIoPerfectHashSeeds: unreferenced. - ManifestData (Core/FIoDependency.cs): unreferenced. - MemoryStreamHelpers: removed Read/Seek/Skip/Tell/Pos extensions that were never called. Only Write extensions remain. Program.cs split from one long Main into focused helpers - EnsureNativeDlls, TryParseEngineVersion, ValidateCliOptions - LogOptionsSummary, TryLoadProvider - BuildManifest, BuildUtocEntryLookup Main/Run now read top-to-bottom as a pipeline. API hygiene - DefaultFileProvider: switch from the (string, SearchOption, bool, VersionContainer?) obsolete ctor to (string, SearchOption, VersionContainer, StringComparer). - FIoStoreTocEntryMeta.ChunkHash marked required so the implicit non-null contract is enforced by the compiler. --- .../Core/{FIoConstants.cs => Constants.cs} | 9 +- UnrealReZen/Core/FIoDependency.cs | 6 - UnrealReZen/Core/FIoPack.cs | 91 +++----- UnrealReZen/Core/FIoStructs.cs | 7 +- UnrealReZen/Core/FIoUToc.cs | 95 -------- .../Core/Helpers/MemoryStreamHelpers.cs | 215 +---------------- UnrealReZen/Core/UToc.cs | 73 ++++++ UnrealReZen/Program.cs | 218 +++++++++++------- 8 files changed, 259 insertions(+), 455 deletions(-) rename UnrealReZen/Core/{FIoConstants.cs => Constants.cs} (67%) delete mode 100644 UnrealReZen/Core/FIoUToc.cs create mode 100644 UnrealReZen/Core/UToc.cs diff --git a/UnrealReZen/Core/FIoConstants.cs b/UnrealReZen/Core/Constants.cs similarity index 67% rename from UnrealReZen/Core/FIoConstants.cs rename to UnrealReZen/Core/Constants.cs index a695c5f..b4bb012 100644 --- a/UnrealReZen/Core/FIoConstants.cs +++ b/UnrealReZen/Core/Constants.cs @@ -1,14 +1,15 @@ -namespace UnrealReZen.Core +namespace UnrealReZen.Core { public static class Constants { - public static string[] CompressionTypes = ["none", "zlib", "oodle", "lz4"]; + public static readonly string[] CompressionTypes = ["none", "zlib", "oodle", "lz4"]; + public static readonly string DefaultAES = "0x" + new string('0', 64); + public static string ToolDirectory = ""; - public static string DefaultAES = "0x" + new string('0', 64); public static string MountPoint = "../../../"; + public const string MagicUtoc = "-==--==--==--==-"; public const string DepFileName = "dependencies"; - public const string UnrealSignature = "\xC1\x83\x2A\x9E"; public const uint NoneEntry = 0xFFFFFFFF; public const int UE5_DepFile_Sig = 1232028526; public const int CompSize = 0x10000; diff --git a/UnrealReZen/Core/FIoDependency.cs b/UnrealReZen/Core/FIoDependency.cs index 327da42..a6bf9e2 100644 --- a/UnrealReZen/Core/FIoDependency.cs +++ b/UnrealReZen/Core/FIoDependency.cs @@ -235,10 +235,4 @@ public void Write(MemoryStream br) } } - public static class ManifestData - { - public const int UE5_DepFile_Sig = 1232028526; - - } - } diff --git a/UnrealReZen/Core/FIoPack.cs b/UnrealReZen/Core/FIoPack.cs index fe37f93..5d8df05 100644 --- a/UnrealReZen/Core/FIoPack.cs +++ b/UnrealReZen/Core/FIoPack.cs @@ -38,27 +38,27 @@ public static int PackToCasToc(string dir, Dependency m, string outFilename, str FilePath = v.Filepath, ChunkID = v.ChunkID, OffLen = offlen, - Metadata = new FIoStoreTocEntryMeta(), + Metadata = new FIoStoreTocEntryMeta { ChunkHash = new FIoChunkHash(new byte[20]) }, CompressionBlocks = [], }; fdata.Add(newEntry); } - fdata.PackFilesToUcas(m, dir, outFilename, compression, depver); + PackFilesToUcas(fdata, m, dir, outFilename, compression, depver); if (!string.Equals(aes.KeyString, Constants.DefaultAES, StringComparison.OrdinalIgnoreCase)) { EncryptUcasInPlace(Path.ChangeExtension(outFilename, ".ucas"), aes.Key); } - var utocBytes = fdata.ConstructUtocFile(compression, aes, gameVer); + var utocBytes = ConstructUtocFile(fdata, compression, aes, gameVer); File.WriteAllBytes(outFilename, utocBytes); File.WriteAllBytes(Path.ChangeExtension(outFilename, ".pak"), PakHolder.Packed_P); return fdata.Count; } - public static void PackFilesToUcas(this List files, Dependency m, string dir, string outFilename, string compression, FIoDependencyFormat depver) + public static void PackFilesToUcas(List files, Dependency m, string dir, string outFilename, string compression, FIoDependencyFormat depver) { var subsetDependencies = new Dictionary(); @@ -229,40 +229,31 @@ private static IEnumerable SplitPath(string path) if (start < path.Length) yield return path[start..]; } - public static byte[] ConstructUtocFile(this List files, string compression, FAesKey AESKey, EGame gameVer) + public static byte[] ConstructUtocFile(List files, string compression, FAesKey aesKey, EGame gameVer) { - var udata = new UToc(new UTocHeader(), [], "", []); + bool isCompressed = !compression.Equals("none", StringComparison.OrdinalIgnoreCase); + bool isEncrypted = !string.Equals(aesKey.KeyString, Constants.DefaultAES, StringComparison.OrdinalIgnoreCase); - var newContainerFlags = (byte)EIoContainerFlags.IndexedContainerFlag; - var compressionMethods = new List { "None" }; + var containerFlags = EIoContainerFlags.IndexedContainerFlag; + if (isCompressed) containerFlags |= EIoContainerFlags.CompressedContainerFlag; + if (isEncrypted) containerFlags |= EIoContainerFlags.EncryptedContainerFlag; - if (!compression.Equals("none", StringComparison.CurrentCultureIgnoreCase)) - { - compressionMethods.Add(compression); - newContainerFlags |= (byte)EIoContainerFlags.CompressedContainerFlag; - } + byte containerChunkType = gameVer >= EGame.GAME_UE5_0 + ? (byte)EIoChunkType5.ContainerHeader + : (byte)EIoChunkType.ContainerHeader; - if (AESKey.KeyString != Constants.DefaultAES) - { - newContainerFlags |= (byte)EIoContainerFlags.EncryptedContainerFlag; - } - - var compressedBlocksCount = 0; - var containerIndex = 0; - - for (var i = 0; i < files.Count; i++) + int compressedBlocksCount = 0; + int containerIndex = 0; + for (int i = 0; i < files.Count; i++) { compressedBlocksCount += files[i].CompressionBlocks.Count; - if ((gameVer < EGame.GAME_UE5_0 && files[i].ChunkID.Type == (byte)EIoChunkType.ContainerHeader) || - gameVer >= EGame.GAME_UE5_0 && files[i].ChunkID.Type == (byte)EIoChunkType5.ContainerHeader) - { - containerIndex = i; - } + if (files[i].ChunkID.Type == containerChunkType) containerIndex = i; } var dirIndexBytes = DeparseDirectoryIndex(files); + uint compressionMethodCount = isCompressed ? 1u : 0u; - udata.HeaderTable = new UTocHeader + var header = new UTocHeader { Magic = Constants.MagicUtoc, Version = (byte)Constants.PackUtocVersion, @@ -270,57 +261,37 @@ public static byte[] ConstructUtocFile(this List files, string co EntryCount = (uint)files.Count, CompressedBlockEntryCount = (uint)compressedBlocksCount, CompressedBlockEntrySize = 12, - CompressionMethodNameCount = (uint)(compressionMethods.Count - 1), + CompressionMethodNameCount = compressionMethodCount, CompressionMethodNameLength = Constants.CompressionNameLength, CompressionBlockSize = Constants.CompSize, DirectoryIndexSize = (uint)dirIndexBytes.Length, ContainerID = new FIoContainerID(files[containerIndex].ChunkID.ID), - ContainerFlags = (EIoContainerFlags)newContainerFlags, + ContainerFlags = containerFlags, PartitionSize = ulong.MaxValue, PartitionCount = 1 }; using var buf = new MemoryStream(); - udata.HeaderTable.Write(buf); - foreach (var file in files) - { - file.ChunkID.Write(buf); - } - foreach (var file in files) - { - file.OffLen.Write(buf); - } + header.Write(buf); + foreach (var file in files) file.ChunkID.Write(buf); + foreach (var file in files) file.OffLen.Write(buf); foreach (var file in files) - { foreach (var block in file.CompressionBlocks) - { block.Write(buf); - } - } - foreach (var compMethod in compressionMethods) + if (isCompressed) { - if (compMethod.Equals("none", StringComparison.CurrentCultureIgnoreCase)) - { - continue; - } - - var capitalized = CultureInfo.CurrentCulture.TextInfo.ToTitleCase(compMethod); - var bname = Encoding.ASCII.GetBytes(capitalized); - var paddedName = new byte[Constants.CompressionNameLength]; - Array.Copy(bname, paddedName, bname.Length); - buf.Write(paddedName, 0, paddedName.Length); + var capitalized = CultureInfo.InvariantCulture.TextInfo.ToTitleCase(compression); + var nameBytes = Encoding.ASCII.GetBytes(capitalized); + var padded = new byte[Constants.CompressionNameLength]; + Array.Copy(nameBytes, padded, nameBytes.Length); + buf.Write(padded, 0, padded.Length); } buf.Write(dirIndexBytes, 0, dirIndexBytes.Length); - - foreach (var file in files) - { - file.Metadata.Write(buf); - } + foreach (var file in files) file.Metadata.Write(buf); return buf.ToArray(); } - } } diff --git a/UnrealReZen/Core/FIoStructs.cs b/UnrealReZen/Core/FIoStructs.cs index fa43835..0b7a9bb 100644 --- a/UnrealReZen/Core/FIoStructs.cs +++ b/UnrealReZen/Core/FIoStructs.cs @@ -160,11 +160,6 @@ public void Write(MemoryStream br) } } - public class FIoPerfectHashSeeds(uint hashSeed) - { - public uint HashSeed { get; set; } = hashSeed; - } - public class FIoChunkID(ulong id, ushort index, byte padding, byte type) { public ulong ID { get; set; } = id; @@ -334,7 +329,7 @@ public void Write(MemoryStream br) public class FIoStoreTocEntryMeta { - public FIoChunkHash ChunkHash { get; set; } + public required FIoChunkHash ChunkHash { get; set; } public FIoStoreTocEntryMetaFlags Flags { get; set; } public void Write(MemoryStream br) { diff --git a/UnrealReZen/Core/FIoUToc.cs b/UnrealReZen/Core/FIoUToc.cs deleted file mode 100644 index c1d5679..0000000 --- a/UnrealReZen/Core/FIoUToc.cs +++ /dev/null @@ -1,95 +0,0 @@ -using CUE4Parse.UE4.Objects.Core.Misc; -using System.Text; -using UnrealReZen.Core.Helpers; - -namespace UnrealReZen.Core -{ - public class UToc(UTocHeader uTocHeader, List gameFileMetaDatas, string mountPoint, List supportedCompressionMethods) - { - public Stream TocStream; - public byte[] AESKey; - public string UCasPath; - public UTocHeader HeaderTable = uTocHeader; - public List ChunkIdsTable; - public List OffsetAndLengthsTable; - public List PerfectHashSeedsTable; - public List CompressionBlocksDataTable; - public List SupportedCompressionMethods = supportedCompressionMethods; - public byte[] SigBlock; - public string MountPoint = mountPoint; - public List OrderedPaths; - public List ChunksMeta; - public List Files = gameFileMetaDatas; - public byte[] aesKey; - } - - public class UTocHeader - { - public string Magic; - public byte Version; - public byte[] Reserved0 = new byte[3]; - public uint HeaderSize; - public uint EntryCount; - public uint CompressedBlockEntryCount; - public uint CompressedBlockEntrySize; - public uint CompressionMethodNameCount; - public uint CompressionMethodNameLength; - public uint CompressionBlockSize; - public uint DirectoryIndexSize; - public uint PartitionCount; - public FIoContainerID ContainerID; - public FGuid EncryptionKeyGuid; - public EIoContainerFlags ContainerFlags; - public byte[] Reserved1 = new byte[3]; - public uint TocChunkPerfectHashSeedsCount; - public ulong PartitionSize; - public uint TocChunksWithoutPerfectHashCount; - public byte[] Reserved2 = new byte[44]; - - public void Write(MemoryStream br) - { - br.Write(Magic, Encoding.UTF8); - br.Write(Version); - br.Write(Reserved0); - br.Write(HeaderSize); - br.Write(EntryCount); - br.Write(CompressedBlockEntryCount); - br.Write(CompressedBlockEntrySize); - br.Write(CompressionMethodNameCount); - br.Write(CompressionMethodNameLength); - br.Write(CompressionBlockSize); - br.Write(DirectoryIndexSize); - br.Write(PartitionCount); - br.Write(ContainerID.Value); - br.Write(0); - br.Write(0); - br.Write(0); - br.Write(0); - br.Write((byte)ContainerFlags); - br.Write(Reserved1); - br.Write(TocChunkPerfectHashSeedsCount); - br.Write(PartitionSize); - br.Write(TocChunksWithoutPerfectHashCount); - br.Write(Reserved2); - - - } - - public static int SizeOf => 144; - } - - public class AssetMetadata - { - public required string FilePath; - public required FIoChunkID ChunkID; - public required FIoOffsetAndLength OffLen; - public required List CompressionBlocks; - public required FIoStoreTocEntryMeta Metadata; - } - - public class FIoContainerID(ulong value) - { - public ulong Value { get; set; } = value; - } -} - diff --git a/UnrealReZen/Core/Helpers/MemoryStreamHelpers.cs b/UnrealReZen/Core/Helpers/MemoryStreamHelpers.cs index 0e7dc85..58e9ac4 100644 --- a/UnrealReZen/Core/Helpers/MemoryStreamHelpers.cs +++ b/UnrealReZen/Core/Helpers/MemoryStreamHelpers.cs @@ -1,214 +1,23 @@ -using System; -using System.Collections.Generic; -using System.IO; using System.Text; namespace UnrealReZen.Core.Helpers { - - public static class MemoryStreamOverrides + public static class MemoryStreamExtensions { + public static void Write(this MemoryStream ms, byte[] input) => ms.Write(input, 0, input.Length); - #region Write Method - //Write Method - public static void Write(this MemoryStream ms, byte[] input) - { - ms.Write(input, 0, input.Length); - } - - public static void Write(this MemoryStream ms,string input, Encoding encoding) - { - ms.Write(encoding.GetBytes(input), 0, encoding.GetBytes(input).Length); - } - - public static void Write(this MemoryStream ms,double input) - { - ms.Write(BitConverter.GetBytes(input), 0, 8); - } - - public static void Write(this MemoryStream ms,float input) - { - ms.Write(BitConverter.GetBytes(input), 0, 4); - } - - public static void Write(this MemoryStream ms,long input) - { - ms.Write(BitConverter.GetBytes(input), 0, 8); - } - - public static void Write(this MemoryStream ms,ulong input) - { - ms.Write(BitConverter.GetBytes(input), 0, 8); - } - - public static void Write(this MemoryStream ms,int input) - { - ms.Write(BitConverter.GetBytes(input), 0, 4); - } - - public static void Write(this MemoryStream ms,uint input) - { - ms.Write(BitConverter.GetBytes(input), 0, 4); - - } - - public static void Write(this MemoryStream ms,short input) - { - ms.Write(BitConverter.GetBytes(input), 0, 2); - } - - public static void Write(this MemoryStream ms,ushort input) - { - ms.Write(BitConverter.GetBytes(input), 0, 2); - - } - - public static void Write(this MemoryStream ms,byte input) - { - ms.WriteByte(input); - } - - #endregion - - #region Read Method - //Read Method - public static byte[] ReadBytes(this MemoryStream ms, int size) - { - var data = new byte[size]; - ms.Read(data, 0, size); - return data; - } - - public static byte[] ReadToEnd(this MemoryStream ms) - { - var data = new byte[ms.Length - ms.Position]; - ms.Read(data, 0, (int)(ms.Length - ms.Position)); - return data; - } - - public static byte[] ReadBytes(this MemoryStream ms, long size) - { - var data = new byte[size]; - ms.Read(data, 0, (int)size); - return data; - } - - public static short ReadInt16(this MemoryStream ms) - { - var data = new byte[2]; - ms.Read(data, 0, 2); - return BitConverter.ToInt16(data, 0); - } - - public static ushort ReadUInt16(this MemoryStream ms) - { - var data = new byte[2]; - ms.Read(data, 0, 2); - return BitConverter.ToUInt16(data, 0); - } - - public static int ReadInt32(this MemoryStream ms) - { - var data = new byte[4]; - ms.Read(data, 0, 4); - return BitConverter.ToInt32(data, 0); - } - - public static uint ReadUInt32(this MemoryStream ms) - { - var data = new byte[4]; - ms.Read(data, 0, 4); - return BitConverter.ToUInt32(data, 0); - } - - public static long ReadInt64(this MemoryStream ms) - { - var data = new byte[8]; - ms.Read(data, 0, 8); - return BitConverter.ToInt64(data, 0); - } - - public static ulong ReadUInt64(this MemoryStream ms) - { - var data = new byte[8]; - ms.Read(data, 0, 8); - return BitConverter.ToUInt64(data, 0); - } - - public static float ReadSingle(this MemoryStream ms) - { - var data = new byte[4]; - ms.Read(data, 0, 4); - return BitConverter.ToSingle(data, 0); - } - - public static string ReadString(this MemoryStream ms, int size, Encoding encoding) - { - var data = new byte[size]; - ms.Read(data, 0, size); - return encoding.GetString(data); - } - - public static string ReadString(this MemoryStream ms) - { - List output = new List(); - while (true) - { - byte reader = (byte)ms.ReadByte(); - if (reader == 0) - { - output.Add(reader); - break; - } - output.Add(reader); - } - return Encoding.ASCII.GetString(output.ToArray()); - } - - public static string ReadStringUTF8(this MemoryStream ms) - { - List output = new List(); - while (true) - { - byte reader = (byte)ms.ReadByte(); - if (reader == 0) - { - output.Add(reader); - break; - } - output.Add(reader); - } - return Encoding.UTF8.GetString(output.ToArray()); - } - #endregion - - #region Location Method - //Location Method - public static void Skip(this MemoryStream ms,int to) - { - ms.Seek(to, SeekOrigin.Current); - } - - public static void Skip(this MemoryStream ms, long to) - { - ms.Seek(to, SeekOrigin.Current); - } - - - public static long Tell(this MemoryStream ms) + public static void Write(this MemoryStream ms, string input, Encoding encoding) { - return ms.Position; + var bytes = encoding.GetBytes(input); + ms.Write(bytes, 0, bytes.Length); } - public static void Seek(this MemoryStream ms, long to) - { - ms.Seek(to, SeekOrigin.Begin); - } - - public static void Pos(this MemoryStream ms, int Base) - { - ms.Position = Base; - } - #endregion + public static void Write(this MemoryStream ms, byte input) => ms.WriteByte(input); + public static void Write(this MemoryStream ms, ushort input) => ms.Write(BitConverter.GetBytes(input), 0, 2); + public static void Write(this MemoryStream ms, short input) => ms.Write(BitConverter.GetBytes(input), 0, 2); + public static void Write(this MemoryStream ms, uint input) => ms.Write(BitConverter.GetBytes(input), 0, 4); + public static void Write(this MemoryStream ms, int input) => ms.Write(BitConverter.GetBytes(input), 0, 4); + public static void Write(this MemoryStream ms, ulong input) => ms.Write(BitConverter.GetBytes(input), 0, 8); + public static void Write(this MemoryStream ms, long input) => ms.Write(BitConverter.GetBytes(input), 0, 8); } } diff --git a/UnrealReZen/Core/UToc.cs b/UnrealReZen/Core/UToc.cs new file mode 100644 index 0000000..9702553 --- /dev/null +++ b/UnrealReZen/Core/UToc.cs @@ -0,0 +1,73 @@ +using CUE4Parse.UE4.Objects.Core.Misc; +using System.Text; +using UnrealReZen.Core.Helpers; + +namespace UnrealReZen.Core +{ + public class UTocHeader + { + public required string Magic { get; init; } + public required byte Version { get; init; } + public byte[] Reserved0 { get; init; } = new byte[3]; + public required uint HeaderSize { get; init; } + public required uint EntryCount { get; init; } + public required uint CompressedBlockEntryCount { get; init; } + public required uint CompressedBlockEntrySize { get; init; } + public required uint CompressionMethodNameCount { get; init; } + public required uint CompressionMethodNameLength { get; init; } + public required uint CompressionBlockSize { get; init; } + public required uint DirectoryIndexSize { get; init; } + public required uint PartitionCount { get; init; } + public required FIoContainerID ContainerID { get; init; } + public FGuid EncryptionKeyGuid { get; init; } + public required EIoContainerFlags ContainerFlags { get; init; } + public byte[] Reserved1 { get; init; } = new byte[3]; + public uint TocChunkPerfectHashSeedsCount { get; init; } + public required ulong PartitionSize { get; init; } + public uint TocChunksWithoutPerfectHashCount { get; init; } + public byte[] Reserved2 { get; init; } = new byte[44]; + + public const int SizeOf = 144; + + public void Write(MemoryStream ms) + { + ms.Write(Magic, Encoding.UTF8); + ms.Write(Version); + ms.Write(Reserved0); + ms.Write(HeaderSize); + ms.Write(EntryCount); + ms.Write(CompressedBlockEntryCount); + ms.Write(CompressedBlockEntrySize); + ms.Write(CompressionMethodNameCount); + ms.Write(CompressionMethodNameLength); + ms.Write(CompressionBlockSize); + ms.Write(DirectoryIndexSize); + ms.Write(PartitionCount); + ms.Write(ContainerID.Value); + ms.Write(0); + ms.Write(0); + ms.Write(0); + ms.Write(0); + ms.Write((byte)ContainerFlags); + ms.Write(Reserved1); + ms.Write(TocChunkPerfectHashSeedsCount); + ms.Write(PartitionSize); + ms.Write(TocChunksWithoutPerfectHashCount); + ms.Write(Reserved2); + } + } + + public class AssetMetadata + { + public required string FilePath { get; set; } + public required FIoChunkID ChunkID { get; init; } + public required FIoOffsetAndLength OffLen { get; init; } + public required List CompressionBlocks { get; init; } + public required FIoStoreTocEntryMeta Metadata { get; init; } + } + + public class FIoContainerID(ulong value) + { + public ulong Value { get; } = value; + } +} diff --git a/UnrealReZen/Program.cs b/UnrealReZen/Program.cs index 753bbe7..715c3ef 100644 --- a/UnrealReZen/Program.cs +++ b/UnrealReZen/Program.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using CommandLine.Text; using CUE4Parse.Compression; using CUE4Parse.Encryption.Aes; @@ -14,7 +14,7 @@ namespace UnrealReZen { - class Options + internal class Options { [Option('g', "game-dir", Required = true, HelpText = "Path to the game directory (for loading UCAS and UTOC files).")] public required string GameDirectory { get; set; } @@ -32,10 +32,10 @@ class Options public string? AESKey { get; set; } [Option("compression-format", Required = false, Default = "Zlib", HelpText = "Compression format (None, Zlib, Oodle, LZ4).")] - public string CompressionFormat { get; set; } + public string CompressionFormat { get; set; } = "Zlib"; [Option("mount-point", Required = false, Default = "../../../", HelpText = "Mount point of packed archive")] - public string MountPoint { get; set; } + public string MountPoint { get; set; } = "../../../"; [Option("container-id", Required = false, HelpText = "Container Id of packed archive (default is a random 8-byte number)")] public ulong? ContainerId { get; set; } @@ -44,136 +44,183 @@ class Options public bool GameDirTopOnly { get; set; } [Usage(ApplicationAlias = "UnrealReZen.exe")] - public static IEnumerable Examples - { - get + public static IEnumerable Examples => [ + new("Making a patch for a ue5 game", new Options { - return [ - new("Making a patch for a ue5 game", new Options { GameDirectory = "C:/Games/MyGame",ContentPath = "C:/Games/MyGame/ExportedFiles", EngineVersion = "GAME_UE5_1", CompressionFormat = "Zlib", OutputPath = "C:/Games/MyGame/TestPatch_P.utoc"}) - ]; - } - } - + GameDirectory = "C:/Games/MyGame", + ContentPath = "C:/Games/MyGame/ExportedFiles", + EngineVersion = "GAME_UE5_1", + CompressionFormat = "Zlib", + OutputPath = "C:/Games/MyGame/TestPatch_P.utoc" + }) + ]; } - internal class Program + + internal static class Program { - static void Main(string[] args) + private const int ExitOk = 0; + private const int ExitError = 1; + + static int Main(string[] args) { Log.Logger = new LoggerConfiguration() - .MinimumLevel.Debug() - .WriteTo.Console() - .CreateLogger(); + .MinimumLevel.Debug() + .WriteTo.Console() + .CreateLogger(); + int exitCode = ExitError; Parser.Default.ParseArguments(args) - .WithParsed(RunOptionsAndReturnExitCode); + .WithParsed(opts => exitCode = Run(opts)); + return exitCode; } - static void RunOptionsAndReturnExitCode(Options opts) + private static int Run(Options opts) { Constants.ToolDirectory = AppDomain.CurrentDomain.BaseDirectory; + + if (!EnsureNativeDlls()) return ExitError; + if (!TryParseEngineVersion(opts.EngineVersion, out var engineVersion)) return ExitError; + if (!ValidateCliOptions(opts)) return ExitError; + + LogOptionsSummary(opts, engineVersion); + + var aesKey = new FAesKey(opts.AESKey ?? Constants.DefaultAES); + if (!TryLoadProvider(opts, engineVersion, aesKey, out var provider)) return ExitError; + + foreach (var vfs in provider.MountedVfs) + { + vfs.Dispose(); + } + + var filesToRepack = Directory.GetFiles(opts.ContentPath, "*", SearchOption.AllDirectories); + if (filesToRepack.Length == 0) + { + Log.Fatal("No valid files found in the content path"); + return ExitError; + } + + Log.Information("Packing Contents..."); + var manifest = BuildManifest(provider, opts, engineVersion, filesToRepack); + + Log.Information("Packing files..."); + Packer.PackToCasToc(opts.ContentPath, manifest, opts.OutputPath, opts.CompressionFormat, aesKey, opts.MountPoint, engineVersion); + Console.WriteLine($"Done! {filesToRepack.Length} file(s) packed"); + return ExitOk; + } + + private static bool EnsureNativeDlls() + { string? oodlePath = Path.Combine(Constants.ToolDirectory, OodleHelper.OodleFileName); if (!OodleHelper.DownloadOodleDll(ref oodlePath)) { - Log.Fatal($"UnrealReZen failed to download the oodle dll. please check you internet connection or place {OodleHelper.OodleFileName} in the tool directory"); - Console.ReadLine(); - return; + Log.Fatal($"UnrealReZen failed to download the oodle dll. please check your internet connection or place {OodleHelper.OodleFileName} in the tool directory"); + return false; } if (!ZlibHelper.DownloadDll(Path.Combine(Constants.ToolDirectory, ZlibHelper.DLL_NAME))) { - Log.Fatal("UnrealReZen failed to download the zlib dll. please check you internet connection or place zlib-ng2.dll in the tool directory"); - Console.ReadLine(); - return; + Log.Fatal($"UnrealReZen failed to download the zlib dll. please check your internet connection or place {ZlibHelper.DLL_NAME} in the tool directory"); + return false; } - if (!Enum.TryParse(typeof(EGame), opts.EngineVersion, out var engineVersion)) + return true; + } + + private static bool TryParseEngineVersion(string raw, out EGame engineVersion) + { + if (Enum.TryParse(raw, out engineVersion) && Enum.IsDefined(engineVersion)) { - Log.Fatal("Invalid Unreal Engine version. Please enter a valid version (e.g., GAME_UE4_0)."); - Log.Information("List of supported engine versions: " + string.Join("\n", Enum.GetNames(typeof(EGame)))); - return; + return true; } - if (!Constants.CompressionTypes.Contains(opts.CompressionFormat.ToLower())) + Log.Fatal("Invalid Unreal Engine version. Please enter a valid version (e.g., GAME_UE4_0)."); + Log.Information("List of supported engine versions: " + string.Join("\n", Enum.GetNames())); + return false; + } + + private static bool ValidateCliOptions(Options opts) + { + if (!Constants.CompressionTypes.Contains(opts.CompressionFormat.ToLowerInvariant())) { Log.Fatal($"Unsupported compression format : {opts.CompressionFormat}"); - return; + return false; } - if (Path.GetExtension(opts.OutputPath) != ".utoc") + if (!string.Equals(Path.GetExtension(opts.OutputPath), ".utoc", StringComparison.OrdinalIgnoreCase)) { - Log.Fatal($"Output path must contains utoc extension"); - return; + Log.Fatal("Output path must contain utoc extension"); + return false; } + return true; + } + private static void LogOptionsSummary(Options opts, EGame engineVersion) + { Console.WriteLine($"Game Directory: {opts.GameDirectory}"); Console.WriteLine($"Content Path: {opts.ContentPath}"); Console.WriteLine($"Unreal Engine Version: {engineVersion}"); Console.WriteLine($"Output Path: {opts.OutputPath}"); + } - var aesKey = new FAesKey(opts.AESKey ?? Constants.DefaultAES); - DefaultFileProvider provider; - + private static bool TryLoadProvider(Options opts, EGame engineVersion, FAesKey aesKey, out DefaultFileProvider provider) + { Log.Information("Loading Game Archives..."); try { var searchOption = opts.GameDirTopOnly ? SearchOption.TopDirectoryOnly : SearchOption.AllDirectories; - provider = new DefaultFileProvider(opts.GameDirectory, searchOption, true, new VersionContainer((EGame)engineVersion)); + provider = new DefaultFileProvider(opts.GameDirectory, searchOption, new VersionContainer(engineVersion), StringComparer.OrdinalIgnoreCase); provider.Initialize(); provider.SubmitKey(new FGuid(), aesKey); provider.LoadLocalization(ELanguage.English); if (provider.RequiredKeys.Count > 0 && provider.Keys.Count == 0) { - Log.Fatal("Some of archives needs AES Key. Please enter aes key"); - return; + Log.Fatal("Some archives require an AES key. Please provide --aes-key."); + return false; } + return true; } catch (Exception ex) { - Log.Fatal("Error:" + ex.ToString()); + Log.Fatal("Error: " + ex); Log.Information("Maybe changing aes key or engine version helps"); - return; + provider = null!; + return false; } + } - foreach (var vfs in provider.MountedVfs) - { - vfs.Dispose(); - } + private static Dependency BuildManifest(DefaultFileProvider provider, Options opts, EGame engineVersion, string[] filesToRepack) + { + var newContainerId = opts.ContainerId ?? CryptographyHelpers.RandomUlong(); + byte containerChunkType = engineVersion >= EGame.GAME_UE5_0 + ? (byte)EIoChunkType5.ContainerHeader + : (byte)EIoChunkType.ContainerHeader; - Log.Information("Packing Contents..."); - Dependency m = new() { Deps = new DependenciesData { ChunkIDToDependencies = [] }, Files = [] }; - List FilesToRepack = new(Directory.GetFiles(opts.ContentPath, "*", SearchOption.AllDirectories)); - var newContainerID = opts.ContainerId ?? CryptographyHelpers.RandomUlong(); - - byte type = (EGame)engineVersion >= EGame.GAME_UE5_0 ? (byte)EIoChunkType5.ContainerHeader : (byte)EIoChunkType.ContainerHeader; - m.Files.Add(new ManifestFile { ChunkID = new FIoChunkID(newContainerID, 0, 0, type), Filepath = Constants.DepFileName }); - m.Deps.ThisPackageID = newContainerID; - if (FilesToRepack.Count == 0) - { - Log.Fatal("No valid files found in the content path"); - return; - } - var utocEntryLookup = new Dictionary>(StringComparer.OrdinalIgnoreCase); - foreach (var entry in provider.Files.Values.OfType()) + var manifest = new Dependency { - if (Path.GetExtension(((AbstractVfsReader)entry.Vfs).Name) != ".utoc") continue; - if (!utocEntryLookup.TryGetValue(entry.Path, out var list)) + Deps = new DependenciesData { - list = new List(); - utocEntryLookup[entry.Path] = list; - } - list.Add(entry); - } + ThisPackageID = newContainerId, + ChunkIDToDependencies = [] + }, + Files = [new ManifestFile + { + ChunkID = new FIoChunkID(newContainerId, 0, 0, containerChunkType), + Filepath = Constants.DepFileName + }] + }; + + var utocEntryLookup = BuildUtocEntryLookup(provider); - foreach (var file in FilesToRepack) + foreach (var file in filesToRepack) { string filename = Path.GetRelativePath(opts.ContentPath, file).Replace('\\', '/'); Log.Information("Mounting " + Path.GetFileName(filename)); if (!utocEntryLookup.TryGetValue(filename, out var matches)) { - Log.Warning("Skipping " + filename + " because its not found in archives."); + Log.Warning("Skipping " + filename + " because it's not found in archives."); continue; } foreach (var entry in matches) { - FIoChunkId chunkId = entry.ChunkId; - m.Files.Add(new ManifestFile + var chunkId = entry.ChunkId; + manifest.Files.Add(new ManifestFile { Filepath = entry.Path, ChunkID = new FIoChunkID(chunkId.ChunkId, 0, 0, chunkId.ChunkType) @@ -184,19 +231,28 @@ static void RunOptionsAndReturnExitCode(Options opts) { foreach (var storeEntry in header.StoreEntries) { - if (!m.Deps.ChunkIDToDependencies.ContainsKey(chunkId.ChunkId)) - { - m.Deps.ChunkIDToDependencies.Add(chunkId.ChunkId, storeEntry); - } + manifest.Deps.ChunkIDToDependencies.TryAdd(chunkId.ChunkId, storeEntry); } } } } - Log.Information("Packing files..."); - Packer.PackToCasToc(opts.ContentPath, m, opts.OutputPath, opts.CompressionFormat, aesKey, opts.MountPoint, (EGame)engineVersion); - Console.WriteLine($"Done! {FilesToRepack.Count} file(s) packed"); + return manifest; + } + private static Dictionary> BuildUtocEntryLookup(DefaultFileProvider provider) + { + var lookup = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var entry in provider.Files.Values.OfType()) + { + if (Path.GetExtension(((AbstractVfsReader)entry.Vfs).Name) != ".utoc") continue; + if (!lookup.TryGetValue(entry.Path, out var list)) + { + list = new List(); + lookup[entry.Path] = list; + } + list.Add(entry); + } + return lookup; } } - } From e5738473e4e42f50f195a20a6bf9265d4fe0ae42 Mon Sep 17 00:00:00 2001 From: NoobInCoding Date: Sat, 18 Apr 2026 19:58:30 +0330 Subject: [PATCH 5/8] Ignore .claude directory Local Claude Code settings (.claude/settings.local.json) were tracked by accident in the initial snapshot. They are per-user, per-machine data and shouldn't live in the repo. Added .claude/ to .gitignore and untracked the existing file. --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 8827004..f32c150 100644 --- a/.gitignore +++ b/.gitignore @@ -399,3 +399,6 @@ FodyWeavers.xsd /UnrealReZen/Properties/launchSettings.json /UnrealReZen/Properties/PublishProfiles/FolderProfile.pubxml /UnrealReZen/Properties/PublishProfiles/FolderProfile.pubxml.user + +# Claude Code +.claude/ From 0c9730a3465e3466e77e3512383a91a089c57477 Mon Sep 17 00:00:00 2001 From: NoobInCoding Date: Sun, 19 Apr 2026 00:22:06 +0330 Subject: [PATCH 6/8] Fix Oodle/Zlib "not initialized" when reading source archives (#60) Symptom - "CUE4Parse.Compression.OodleException: Oodle decompression failed: not initialized" at provider load time, even though oo2core / oodle dll was present next to the exe. Users had to manually register the instance from their own code to work around it. Root cause - EnsureNativeDlls was calling OodleHelper.DownloadOodleDll and ZlibHelper.DownloadDll, which only *fetch* the DLL. Neither sets OodleHelper.Instance / ZlibHelper.Instance, so CUE4Parse's decoder threw the moment it tried to read an Oodle/Zlib-compressed source container. Fix - Use OodleHelper.Initialize / ZlibHelper.Initialize, which download (if missing) AND register the Instance. Verify Instance is non-null afterwards and fail fast with a clear message if not. --- UnrealReZen/Program.cs | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/UnrealReZen/Program.cs b/UnrealReZen/Program.cs index 715c3ef..b6be4a1 100644 --- a/UnrealReZen/Program.cs +++ b/UnrealReZen/Program.cs @@ -110,15 +110,33 @@ private static int Run(Options opts) private static bool EnsureNativeDlls() { - string? oodlePath = Path.Combine(Constants.ToolDirectory, OodleHelper.OodleFileName); - if (!OodleHelper.DownloadOodleDll(ref oodlePath)) + try + { + OodleHelper.Initialize(Path.Combine(Constants.ToolDirectory, OodleHelper.OodleFileName)); + } + catch (Exception ex) + { + Log.Fatal(ex, $"Failed to initialize Oodle. Place {OodleHelper.OodleFileName} in the tool directory and try again."); + return false; + } + if (OodleHelper.Instance is null) + { + Log.Fatal($"Oodle was not registered. Place {OodleHelper.OodleFileName} in the tool directory or check your internet connection."); + return false; + } + + try + { + ZlibHelper.Initialize(Path.Combine(Constants.ToolDirectory, ZlibHelper.DllName)); + } + catch (Exception ex) { - Log.Fatal($"UnrealReZen failed to download the oodle dll. please check your internet connection or place {OodleHelper.OodleFileName} in the tool directory"); + Log.Fatal(ex, $"Failed to initialize Zlib. Place {ZlibHelper.DllName} in the tool directory and try again."); return false; } - if (!ZlibHelper.DownloadDll(Path.Combine(Constants.ToolDirectory, ZlibHelper.DLL_NAME))) + if (ZlibHelper.Instance is null) { - Log.Fatal($"UnrealReZen failed to download the zlib dll. please check your internet connection or place {ZlibHelper.DLL_NAME} in the tool directory"); + Log.Fatal($"Zlib was not registered. Place {ZlibHelper.DllName} in the tool directory or check your internet connection."); return false; } return true; From e60c33479d683f4867f3bed0908b328b91e34264 Mon Sep 17 00:00:00 2001 From: NoobInCoding Date: Sun, 19 Apr 2026 00:23:17 +0330 Subject: [PATCH 7/8] Decouple --aes-key from output encryption; add --encrypt-output opt-in (#53) Symptom - Passing --aes-key (required to read encrypted source archives) also silently encrypted the output .ucas and set the EncryptedContainerFlag in the .utoc. The tool only encrypted the .ucas body though -- it never encrypted the .utoc directory index -- so the resulting container was structurally invalid for games that honor the flag (FModel flagged "IsEncrypted for no reason", games crashed on load). Design change - --aes-key is now strictly "key for decrypting the source game". - New --encrypt-output flag (default false) opts in to encrypting the generated .ucas and setting the EncryptedContainerFlag. Most UE4/5 games accept plain archives for mods, so this matches the common case and the behavior of community forks (GhostyPool, matyamod). Code - Packer.PackToCasToc now takes a nullable FAesKey? outputAes. Null means "don't encrypt, don't set the flag"; non-null means "encrypt UCAS with this key and set the flag". - ConstructUtocFile takes an explicit bool isEncrypted instead of inferring it from the AES key string. - Program.Run only forwards the key when opts.EncryptOutput is true. Known limitation - When --encrypt-output is passed, the .utoc directory index itself is still not encrypted. Games that strictly require an encrypted directory index will reject the container. Proper directory-index encryption is tracked separately; this commit fixes the common "don't encrypt at all" path. --- UnrealReZen/Core/FIoPack.cs | 11 +++++------ UnrealReZen/Program.cs | 8 ++++++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/UnrealReZen/Core/FIoPack.cs b/UnrealReZen/Core/FIoPack.cs index 5d8df05..9832c26 100644 --- a/UnrealReZen/Core/FIoPack.cs +++ b/UnrealReZen/Core/FIoPack.cs @@ -12,7 +12,7 @@ namespace UnrealReZen.Core { public static class Packer { - public static int PackToCasToc(string dir, Dependency m, string outFilename, string compression, FAesKey aes, string mountPoint, EGame gameVer) + public static int PackToCasToc(string dir, Dependency m, string outFilename, string compression, FAesKey? outputAes, string mountPoint, EGame gameVer) { FIoDependencyFormat depver = gameVer >= EGame.GAME_UE5_0 ? FIoDependencyFormat.UE5 : FIoDependencyFormat.UE4; @@ -47,12 +47,12 @@ public static int PackToCasToc(string dir, Dependency m, string outFilename, str PackFilesToUcas(fdata, m, dir, outFilename, compression, depver); - if (!string.Equals(aes.KeyString, Constants.DefaultAES, StringComparison.OrdinalIgnoreCase)) + if (outputAes is not null) { - EncryptUcasInPlace(Path.ChangeExtension(outFilename, ".ucas"), aes.Key); + EncryptUcasInPlace(Path.ChangeExtension(outFilename, ".ucas"), outputAes.Key); } - var utocBytes = ConstructUtocFile(fdata, compression, aes, gameVer); + var utocBytes = ConstructUtocFile(fdata, compression, outputAes is not null, gameVer); File.WriteAllBytes(outFilename, utocBytes); File.WriteAllBytes(Path.ChangeExtension(outFilename, ".pak"), PakHolder.Packed_P); return fdata.Count; @@ -229,10 +229,9 @@ private static IEnumerable SplitPath(string path) if (start < path.Length) yield return path[start..]; } - public static byte[] ConstructUtocFile(List files, string compression, FAesKey aesKey, EGame gameVer) + public static byte[] ConstructUtocFile(List files, string compression, bool isEncrypted, EGame gameVer) { bool isCompressed = !compression.Equals("none", StringComparison.OrdinalIgnoreCase); - bool isEncrypted = !string.Equals(aesKey.KeyString, Constants.DefaultAES, StringComparison.OrdinalIgnoreCase); var containerFlags = EIoContainerFlags.IndexedContainerFlag; if (isCompressed) containerFlags |= EIoContainerFlags.CompressedContainerFlag; diff --git a/UnrealReZen/Program.cs b/UnrealReZen/Program.cs index b6be4a1..0cae02d 100644 --- a/UnrealReZen/Program.cs +++ b/UnrealReZen/Program.cs @@ -28,9 +28,12 @@ internal class Options [Option('o', "output-path", Required = true, HelpText = "Path (including file name) for the packed utoc file.")] public required string OutputPath { get; set; } - [Option('a', "aes-key", Required = false, HelpText = "AES key of the game (only if its encrypted)")] + [Option('a', "aes-key", Required = false, HelpText = "AES key for reading the game's encrypted source archives. Not used to encrypt the output unless --encrypt-output is also set.")] public string? AESKey { get; set; } + [Option("encrypt-output", Required = false, Default = false, HelpText = "Encrypt the generated .ucas with the game's AES key and set the EncryptedContainerFlag in the .utoc. Most games accept plain archives for mods; leave off unless you know the target refuses unencrypted containers.")] + public bool EncryptOutput { get; set; } + [Option("compression-format", Required = false, Default = "Zlib", HelpText = "Compression format (None, Zlib, Oodle, LZ4).")] public string CompressionFormat { get; set; } = "Zlib"; @@ -103,7 +106,8 @@ private static int Run(Options opts) var manifest = BuildManifest(provider, opts, engineVersion, filesToRepack); Log.Information("Packing files..."); - Packer.PackToCasToc(opts.ContentPath, manifest, opts.OutputPath, opts.CompressionFormat, aesKey, opts.MountPoint, engineVersion); + var outputAesKey = opts.EncryptOutput ? aesKey : null; + Packer.PackToCasToc(opts.ContentPath, manifest, opts.OutputPath, opts.CompressionFormat, outputAesKey, opts.MountPoint, engineVersion); Console.WriteLine($"Done! {filesToRepack.Length} file(s) packed"); return ExitOk; } From 5e2534d130ae3cfeea77e86b700b921607e23db2 Mon Sep 17 00:00:00 2001 From: NoobInCoding Date: Sun, 19 Apr 2026 00:36:46 +0330 Subject: [PATCH 8/8] CI: target .NET 8, fetch the CUE4Parse submodule, drop phantom tests What changed and why - .NET 7 -> 8. The project now targets net8.0 (csproj), so the old setup-dotnet@v3 with 7.0.x would fail to restore. - Checkout with submodules: false, then an explicit "git submodule update --init external/CUE4Parse". A plain submodules: recursive dies on a broken catch2 pin buried under ACL/rtm's transitive submodules; we don't need those anyway. CUE4Parse's native CMake step is non-fatal (it logs a warning and continues if the natives don't build), so skipping ACL is safe - our tool downloads its own Oodle/Zlib DLLs at runtime. - Dropped the "dotnet test" step. There is no test project in the repo, so the step was always a no-op-or-fail depending on SDK version. - Bumped actions/checkout v3 -> v4, actions/setup-dotnet v3 -> v4. - Removed the **.cs / **.csproj path filter. A change to the workflow file, .gitmodules, or the submodule pin should also retrigger CI. - Matrix left with a single entry (windows-latest) but kept as a matrix so future platforms can be added without restructuring. --- .github/workflows/dotnet-desktop.yml | 54 ++++++++++++++-------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/.github/workflows/dotnet-desktop.yml b/.github/workflows/dotnet-desktop.yml index 0ff801a..425f982 100644 --- a/.github/workflows/dotnet-desktop.yml +++ b/.github/workflows/dotnet-desktop.yml @@ -1,36 +1,38 @@ -name: build and test +name: build on: pull_request: - branches: [ master ] - paths: - - '**.cs' - - '**.csproj' + branches: [master] push: - branches: - - master + branches: [master] workflow_dispatch: jobs: - build-and-test: - - name: build-and-test-${{matrix.os}} - runs-on: windows-latest + build: + name: build-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest] steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up .NET - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '7.0.x' + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + + - name: Init CUE4Parse submodule + shell: bash + run: git submodule update --init external/CUE4Parse + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore + run: dotnet restore UnrealReZen/UnrealReZen.csproj - - name: Restore dependencies - run: dotnet restore - - - name: Build solution - run: dotnet build --no-restore --configuration Release - - - name: Run tests - run: dotnet test --no-build --verbosity normal --configuration Release + - name: Build + run: dotnet build UnrealReZen/UnrealReZen.csproj --no-restore --configuration Release