diff --git a/.github/workflows/dotnet-desktop.yml b/.github/workflows/dotnet-desktop.yml index 0ff801a..425f982 100644 --- a/.github/workflows/dotnet-desktop.yml +++ b/.github/workflows/dotnet-desktop.yml @@ -1,36 +1,38 @@ -name: build and test +name: build on: pull_request: - branches: [ master ] - paths: - - '**.cs' - - '**.csproj' + branches: [master] push: - branches: - - master + branches: [master] workflow_dispatch: jobs: - build-and-test: - - name: build-and-test-${{matrix.os}} - runs-on: windows-latest + build: + name: build-${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-latest] steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Set up .NET - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '7.0.x' + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + + - name: Init CUE4Parse submodule + shell: bash + run: git submodule update --init external/CUE4Parse + + - name: Set up .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Restore + run: dotnet restore UnrealReZen/UnrealReZen.csproj - - name: Restore dependencies - run: dotnet restore - - - name: Build solution - run: dotnet build --no-restore --configuration Release - - - name: Run tests - run: dotnet test --no-build --verbosity normal --configuration Release + - name: Build + run: dotnet build UnrealReZen/UnrealReZen.csproj --no-restore --configuration Release diff --git a/.gitignore b/.gitignore index 8827004..f32c150 100644 --- a/.gitignore +++ b/.gitignore @@ -399,3 +399,6 @@ FodyWeavers.xsd /UnrealReZen/Properties/launchSettings.json /UnrealReZen/Properties/PublishProfiles/FolderProfile.pubxml /UnrealReZen/Properties/PublishProfiles/FolderProfile.pubxml.user + +# Claude Code +.claude/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..a898a2f --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/CUE4Parse"] + path = external/CUE4Parse + url = https://github.com/FabianFG/CUE4Parse.git diff --git a/UnrealReZen/Core/Compression/Compressions.cs b/UnrealReZen/Core/Compression/Compressions.cs index d5789ce..2560a4e 100644 --- a/UnrealReZen/Core/Compression/Compressions.cs +++ b/UnrealReZen/Core/Compression/Compressions.cs @@ -4,9 +4,8 @@ namespace UnrealReZen.Core.Compression { - public class CompressionUtils + public static class CompressionUtils { - public static readonly Dictionary> CompressionMethods = new(StringComparer.OrdinalIgnoreCase) { { "None", CompressNone }, @@ -15,64 +14,53 @@ public class CompressionUtils { "Lz4", CompressLZ4 } }; - public static byte[]? Compress(string method, byte[] inputData) - { - if (CompressionMethods.TryGetValue(method, out var compressionFunction)) - { - return compressionFunction(inputData); - } - return null; - } - public static Func? GetCompressionFunction(string method) - { - if (CompressionMethods.TryGetValue(method, out var compressionFunction)) - { - return compressionFunction; - } - return null; - } + => CompressionMethods.TryGetValue(method, out var fn) ? fn : null; - private static byte[] CompressNone(byte[] inData) - { - return inData; - } + private static readonly Lazy _zlib = new(() => + new Zlibng(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.ZlibHelper.DllName)), + isThreadSafe: true); + + private static readonly Lazy _oodle = new(() => + new Oodle(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.OodleHelper.OodleFileName)), + isThreadSafe: true); + + private static byte[] CompressNone(byte[] inData) => inData; private static byte[] CompressZlib(byte[] inData) { - using var _zlib = new Zlibng(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.ZlibHelper.DLL_NAME)); - var compressedBufferSize = (int)_zlib.CompressBound(inData.Length); - var compressedBuffer = new byte[compressedBufferSize]; - var compressionResult = _zlib.Compress(compressedBuffer, inData, out int compressedSize); - if (compressionResult.CompareTo(ZlibngCompressionResult.Ok) != 0) + var zlib = _zlib.Value; + var bufferSize = (int)zlib.CompressBound(inData.Length); + var buffer = new byte[bufferSize]; + var result = zlib.Compress(buffer, inData, out int compressedSize); + if (result != ZlibngCompressionResult.Ok) { - throw new Exception($"Zlib compression failed with error code {compressionResult}"); + throw new InvalidOperationException($"Zlib compression failed with error code {result}"); } - return compressedBuffer.AsSpan(0, compressedSize).ToArray(); - + Array.Resize(ref buffer, compressedSize); + return buffer; } private static byte[] CompressOodle(byte[] inData) { const OodleCompressor compressor = OodleCompressor.Kraken; - using var _oodle = new Oodle(Path.Combine(Constants.ToolDirectory, CUE4Parse.Compression.OodleHelper.OODLE_DLL_NAME)); - var compressedBufferSize = (int)_oodle.GetCompressedBufferSizeNeeded(compressor, inData.Length); - var compressedBuffer = new byte[compressedBufferSize]; - var compressedSize = (int)_oodle.Compress(compressor, OodleCompressionLevel.Max, inData, compressedBuffer); - return [.. compressedBuffer.Take(compressedSize)]; + var oodle = _oodle.Value; + var bufferSize = (int)oodle.GetCompressedBufferSizeNeeded(compressor, inData.Length); + var buffer = new byte[bufferSize]; + var compressedSize = (int)oodle.Compress(compressor, OodleCompressionLevel.Max, inData, buffer); + Array.Resize(ref buffer, compressedSize); + return buffer; } private static byte[] CompressLZ4(byte[] inData) { - using (var inputStream = new MemoryStream(inData)) - using (var outputStream = new MemoryStream()) + using var inputStream = new MemoryStream(inData); + using var outputStream = new MemoryStream(inData.Length); using (var lz4Stream = LZ4Stream.Encode(outputStream)) { inputStream.CopyTo(lz4Stream); - lz4Stream.Flush(); - return outputStream.ToArray(); } - + return outputStream.ToArray(); } } -} \ No newline at end of file +} diff --git a/UnrealReZen/Core/FIoConstants.cs b/UnrealReZen/Core/Constants.cs similarity index 67% rename from UnrealReZen/Core/FIoConstants.cs rename to UnrealReZen/Core/Constants.cs index a695c5f..b4bb012 100644 --- a/UnrealReZen/Core/FIoConstants.cs +++ b/UnrealReZen/Core/Constants.cs @@ -1,14 +1,15 @@ -namespace UnrealReZen.Core +namespace UnrealReZen.Core { public static class Constants { - public static string[] CompressionTypes = ["none", "zlib", "oodle", "lz4"]; + public static readonly string[] CompressionTypes = ["none", "zlib", "oodle", "lz4"]; + public static readonly string DefaultAES = "0x" + new string('0', 64); + public static string ToolDirectory = ""; - public static string DefaultAES = "0x" + new string('0', 64); public static string MountPoint = "../../../"; + public const string MagicUtoc = "-==--==--==--==-"; public const string DepFileName = "dependencies"; - public const string UnrealSignature = "\xC1\x83\x2A\x9E"; public const uint NoneEntry = 0xFFFFFFFF; public const int UE5_DepFile_Sig = 1232028526; public const int CompSize = 0x10000; diff --git a/UnrealReZen/Core/FIoDependency.cs b/UnrealReZen/Core/FIoDependency.cs index 327da42..a6bf9e2 100644 --- a/UnrealReZen/Core/FIoDependency.cs +++ b/UnrealReZen/Core/FIoDependency.cs @@ -235,10 +235,4 @@ public void Write(MemoryStream br) } } - public static class ManifestData - { - public const int UE5_DepFile_Sig = 1232028526; - - } - } diff --git a/UnrealReZen/Core/FIoPack.cs b/UnrealReZen/Core/FIoPack.cs index a318d75..9832c26 100644 --- a/UnrealReZen/Core/FIoPack.cs +++ b/UnrealReZen/Core/FIoPack.cs @@ -3,7 +3,7 @@ using CUE4Parse.UE4.Objects.Core.Misc; using CUE4Parse.UE4.Versions; using System.Globalization; -using System.IO.MemoryMappedFiles; +using System.Security.Cryptography; using System.Text; using UnrealReZen.Core.Compression; using UnrealReZen.Core.Helpers; @@ -12,7 +12,7 @@ namespace UnrealReZen.Core { public static class Packer { - public static int PackToCasToc(string dir, Dependency m, string outFilename, string compression, FAesKey aes, string mountPoint, EGame gameVer) + public static int PackToCasToc(string dir, Dependency m, string outFilename, string compression, FAesKey? outputAes, string mountPoint, EGame gameVer) { FIoDependencyFormat depver = gameVer >= EGame.GAME_UE5_0 ? FIoDependencyFormat.UE5 : FIoDependencyFormat.UE4; @@ -38,29 +38,27 @@ public static int PackToCasToc(string dir, Dependency m, string outFilename, str FilePath = v.Filepath, ChunkID = v.ChunkID, OffLen = offlen, - Metadata = new FIoStoreTocEntryMeta(), + Metadata = new FIoStoreTocEntryMeta { ChunkHash = new FIoChunkHash(new byte[20]) }, CompressionBlocks = [], }; fdata.Add(newEntry); } - fdata.PackFilesToUcas(m, dir, outFilename, compression, depver); + PackFilesToUcas(fdata, m, dir, outFilename, compression, depver); - if (aes.KeyString != Constants.DefaultAES) + if (outputAes is not null) { - var b = File.ReadAllBytes(Path.ChangeExtension(outFilename, ".ucas")); - var encrypted = CryptographyHelpers.EncryptAES(b, aes.Key); - File.WriteAllBytes(Path.ChangeExtension(outFilename, ".ucas"), encrypted); + EncryptUcasInPlace(Path.ChangeExtension(outFilename, ".ucas"), outputAes.Key); } - var utocBytes = fdata.ConstructUtocFile(compression, aes, gameVer); + var utocBytes = ConstructUtocFile(fdata, compression, outputAes is not null, gameVer); File.WriteAllBytes(outFilename, utocBytes); File.WriteAllBytes(Path.ChangeExtension(outFilename, ".pak"), PakHolder.Packed_P); return fdata.Count; } - public static void PackFilesToUcas(this List files, Dependency m, string dir, string outFilename, string compression, FIoDependencyFormat depver) + public static void PackFilesToUcas(List files, Dependency m, string dir, string outFilename, string compression, FIoDependencyFormat depver) { var subsetDependencies = new Dictionary(); @@ -79,32 +77,21 @@ public static void PackFilesToUcas(this List files, Dependency m, m.Deps.ChunkIDToDependencies = subsetDependencies; - var compMethodNumber = !compression.Equals("none", StringComparison.CurrentCultureIgnoreCase) ? (byte)1 : (byte)0; + var compMethodNumber = !compression.Equals("none", StringComparison.OrdinalIgnoreCase) ? (byte)1 : (byte)0; var compFun = CompressionUtils.GetCompressionFunction(compression) ?? throw new Exception("Could not find " + compression + " method. Please use None, Oodle or Zlib"); using var f = File.Open(Path.ChangeExtension(outFilename, ".ucas"), FileMode.Create); + var readBuffer = new byte[Constants.CompSize]; + Span padBuffer = stackalloc byte[16]; for (int i = 0; i < files.Count; i++) { - WriteProgressBar(i, files.Count - 1); + WriteProgressBar(i + 1, files.Count); - MemoryMappedFile mmf; - long SizeOfmmf; - string pathToread = Path.Combine(dir.Replace("/", "\\"), files[i].FilePath.Replace("/", "\\")); - if (!File.Exists(pathToread)) - { - if (files[i].FilePath != Constants.DepFileName) throw new Exception("File doesn't exist, and also its not the dependency file."); - byte[] ManifestCreatedFile = depver == FIoDependencyFormat.UE4 ? m.WriteDependenciesAsUE4() : m.WriteDependenciesAsUE5(); - mmf = MemoryMappedHelpers.CreateMemoryMappedFileFromByteArray(ManifestCreatedFile, files[i].FilePath); - SizeOfmmf = ManifestCreatedFile.LongLength; - files[i].FilePath = ""; - } - else - { - mmf = MemoryMappedFile.CreateFromFile(pathToread, FileMode.Open, Path.GetFileNameWithoutExtension(pathToread)); - SizeOfmmf = new FileInfo(pathToread).Length; - } + var pathToRead = Path.Combine(dir, files[i].FilePath); + using ChunkSource source = File.Exists(pathToRead) + ? ChunkSource.FromFile(pathToRead) + : CreateDependencyChunkSource(files[i], m, depver); - - files[i].OffLen.SetLength((ulong)SizeOfmmf); + files[i].OffLen.SetLength((ulong)source.Length); if (i == 0) { @@ -117,82 +104,95 @@ public static void PackFilesToUcas(this List files, Dependency m, files[i].OffLen.SetOffset(off); } - files[i].Metadata.ChunkHash = new FIoChunkHash(mmf.SHA1Hash()); files[i].Metadata.Flags = FIoStoreTocEntryMetaFlags.CompressedMetaFlag; - long PosOfReaded = 0; - long RemainSize = SizeOfmmf; - while (PosOfReaded != SizeOfmmf) + using var sha1 = SHA1.Create(); + long readPos = 0; + while (readPos < source.Length) { - var block = new FIoStoreTocCompressedBlockEntry(); - var chunkLen = RemainSize; - if (chunkLen > Constants.CompSize) - { - chunkLen = Constants.CompSize; - } - RemainSize -= chunkLen; - var chunk = mmf.ReadBytesOfFile(PosOfReaded, chunkLen); - PosOfReaded += chunkLen; - var cChunkPtr = compFun(chunk); - var compressedChunk = cChunkPtr.ToArray(); + int chunkLen = (int)Math.Min(Constants.CompSize, source.Length - readPos); + source.ReadInto(readPos, readBuffer, 0, chunkLen); + readPos += chunkLen; + sha1.TransformBlock(readBuffer, 0, chunkLen, readBuffer, 0); + + byte[] compressedChunk = compFun(chunkLen == readBuffer.Length ? readBuffer : readBuffer[..chunkLen]); - block.CompressionMethod = compMethodNumber; + var block = new FIoStoreTocCompressedBlockEntry + { + CompressionMethod = compMethodNumber + }; block.SetOffset((ulong)f.Position); block.SetUncompressedSize((uint)chunkLen); block.SetCompressedSize((uint)compressedChunk.Length); - - compressedChunk = [.. compressedChunk, .. CryptographyHelpers.GetRandomBytes(0x10 - compressedChunk.Length % 0x10 & 0x10 - 1)]; files[i].CompressionBlocks.Add(block); f.Write(compressedChunk, 0, compressedChunk.Length); + int padLen = (0x10 - compressedChunk.Length % 0x10) & 0x0F; + if (padLen > 0) + { + RandomNumberGenerator.Fill(padBuffer[..padLen]); + f.Write(padBuffer[..padLen]); + } } - mmf.Dispose(); + sha1.TransformFinalBlock(readBuffer, 0, 0); + files[i].Metadata.ChunkHash = new FIoChunkHash(sha1.Hash!); } - // Add a line feed for the progress bar Console.WriteLine(""); } - public static void WriteProgressBar(int count, int maxCount) + private static ChunkSource CreateDependencyChunkSource(AssetMetadata entry, Dependency m, FIoDependencyFormat depver) + { + if (entry.FilePath != Constants.DepFileName) + { + throw new FileNotFoundException($"Content file not found: {entry.FilePath}"); + } + byte[] depBytes = depver == FIoDependencyFormat.UE4 ? m.WriteDependenciesAsUE4() : m.WriteDependenciesAsUE5(); + entry.FilePath = ""; + return ChunkSource.FromBytes(depBytes); + } + + private static void EncryptUcasInPlace(string path, byte[] aesKey) + { + string tempPath = path + ".enc.tmp"; + using (var aes = System.Security.Cryptography.Aes.Create()) + { + aes.Key = aesKey; + aes.Mode = CipherMode.ECB; + aes.Padding = PaddingMode.None; + using var input = File.Open(path, FileMode.Open, FileAccess.Read); + using var output = File.Open(tempPath, FileMode.Create, FileAccess.Write); + using var encryptor = aes.CreateEncryptor(); + using var crypto = new CryptoStream(output, encryptor, CryptoStreamMode.Write); + input.CopyTo(crypto); + } + File.Move(tempPath, path, overwrite: true); + } + + public static void WriteProgressBar(int count, int total) { - // Display a progress bar on the console. - // e.g. WriteProgressBar(54, 100) - // [##########..........] 54/100 const int MaxProgress = 20; - var progress = count * MaxProgress / maxCount; + int denom = Math.Max(total, 1); + int progress = Math.Min(count * MaxProgress / denom, MaxProgress); string str = new string('#', progress) + new string('.', MaxProgress - progress); - Console.Write($"\r[{str}] {count}/{maxCount}"); + Console.Write($"\r[{str}] {count}/{total}"); } public static byte[] DeparseDirectoryIndex(List files) { - var wrapper = new DirIndexWrapper(); - var dirIndexEntries = new List(); - var fileIndexEntries = new List(); - - var strmap = new Dictionary(); - foreach (var v in files) + var strIdx = new Dictionary(); + var strSlice = new List(); + foreach (var file in files) { - var dirfiles = v.FilePath.Split('/'); - if (dirfiles[0] == "") + foreach (var segment in SplitPath(file.FilePath)) { - dirfiles = dirfiles.Skip(1).ToArray(); - } - - foreach (var str in dirfiles) - { - strmap[str] = true; + if (strIdx.TryAdd(segment, strSlice.Count)) + { + strSlice.Add(segment); + } } } - var strSlice = strmap.Keys.ToList(); - var strIdx = new Dictionary(); - for (int iv = 0; iv < strSlice.Count; iv++) - { - strIdx.Add(strSlice[iv], iv); - } - - var root = new FIoDirectoryIndexEntry { Name = Constants.NoneEntry, @@ -201,60 +201,58 @@ public static byte[] DeparseDirectoryIndex(List files) FirstFileEntry = Constants.NoneEntry, }; - dirIndexEntries.Add(root); - wrapper.Dirs = dirIndexEntries; - wrapper.Files = fileIndexEntries; - wrapper.StrTable = strIdx; - wrapper.StrSlice = [.. strSlice]; + var wrapper = new DirIndexWrapper( + new List { root }, + new List(files.Count), + strIdx, + strSlice.ToArray()); for (var i = 0; i < files.Count; i++) { - var fpathSections = files[i].FilePath.Split('/'); - if (fpathSections[0] == "") - { - fpathSections = fpathSections.Skip(1).ToArray(); - } - - root.AddFile(fpathSections, (uint)i, wrapper); + root.AddFile(SplitPath(files[i].FilePath).ToArray(), (uint)i, wrapper); } return wrapper.ToBytes(); } - public static byte[] ConstructUtocFile(this List files, string compression, FAesKey AESKey, EGame gameVer) + private static IEnumerable SplitPath(string path) { - var udata = new UToc(new UTocHeader(), [], "", []); - - var newContainerFlags = (byte)EIoContainerFlags.IndexedContainerFlag; - var compressionMethods = new List { "None" }; - - if (!compression.Equals("none", StringComparison.CurrentCultureIgnoreCase)) + int start = path.StartsWith('/') ? 1 : 0; + for (int i = start; i < path.Length; i++) { - compressionMethods.Add(compression); - newContainerFlags |= (byte)EIoContainerFlags.CompressedContainerFlag; + if (path[i] == '/') + { + if (i > start) yield return path[start..i]; + start = i + 1; + } } + if (start < path.Length) yield return path[start..]; + } - if (AESKey.KeyString != Constants.DefaultAES) - { - newContainerFlags |= (byte)EIoContainerFlags.EncryptedContainerFlag; - } + public static byte[] ConstructUtocFile(List files, string compression, bool isEncrypted, EGame gameVer) + { + bool isCompressed = !compression.Equals("none", StringComparison.OrdinalIgnoreCase); - var compressedBlocksCount = 0; - var containerIndex = 0; + var containerFlags = EIoContainerFlags.IndexedContainerFlag; + if (isCompressed) containerFlags |= EIoContainerFlags.CompressedContainerFlag; + if (isEncrypted) containerFlags |= EIoContainerFlags.EncryptedContainerFlag; - for (var i = 0; i < files.Count; i++) + byte containerChunkType = gameVer >= EGame.GAME_UE5_0 + ? (byte)EIoChunkType5.ContainerHeader + : (byte)EIoChunkType.ContainerHeader; + + int compressedBlocksCount = 0; + int containerIndex = 0; + for (int i = 0; i < files.Count; i++) { compressedBlocksCount += files[i].CompressionBlocks.Count; - if ((gameVer < EGame.GAME_UE5_0 && files[i].ChunkID.Type == (byte)EIoChunkType.ContainerHeader) || - gameVer >= EGame.GAME_UE5_0 && files[i].ChunkID.Type == (byte)EIoChunkType5.ContainerHeader) - { - containerIndex = i; - } + if (files[i].ChunkID.Type == containerChunkType) containerIndex = i; } var dirIndexBytes = DeparseDirectoryIndex(files); + uint compressionMethodCount = isCompressed ? 1u : 0u; - udata.HeaderTable = new UTocHeader + var header = new UTocHeader { Magic = Constants.MagicUtoc, Version = (byte)Constants.PackUtocVersion, @@ -262,57 +260,37 @@ public static byte[] ConstructUtocFile(this List files, string co EntryCount = (uint)files.Count, CompressedBlockEntryCount = (uint)compressedBlocksCount, CompressedBlockEntrySize = 12, - CompressionMethodNameCount = (uint)(compressionMethods.Count - 1), + CompressionMethodNameCount = compressionMethodCount, CompressionMethodNameLength = Constants.CompressionNameLength, CompressionBlockSize = Constants.CompSize, DirectoryIndexSize = (uint)dirIndexBytes.Length, ContainerID = new FIoContainerID(files[containerIndex].ChunkID.ID), - ContainerFlags = (EIoContainerFlags)newContainerFlags, + ContainerFlags = containerFlags, PartitionSize = ulong.MaxValue, PartitionCount = 1 }; using var buf = new MemoryStream(); - udata.HeaderTable.Write(buf); + header.Write(buf); + foreach (var file in files) file.ChunkID.Write(buf); + foreach (var file in files) file.OffLen.Write(buf); foreach (var file in files) - { - file.ChunkID.Write(buf); - } - foreach (var file in files) - { - file.OffLen.Write(buf); - } - foreach (var file in files) - { foreach (var block in file.CompressionBlocks) - { block.Write(buf); - } - } - foreach (var compMethod in compressionMethods) + if (isCompressed) { - if (compMethod.Equals("none", StringComparison.CurrentCultureIgnoreCase)) - { - continue; - } - - var capitalized = CultureInfo.CurrentCulture.TextInfo.ToTitleCase(compMethod); - var bname = Encoding.ASCII.GetBytes(capitalized); - var paddedName = new byte[Constants.CompressionNameLength]; - Array.Copy(bname, paddedName, bname.Length); - buf.Write(paddedName, 0, paddedName.Length); + var capitalized = CultureInfo.InvariantCulture.TextInfo.ToTitleCase(compression); + var nameBytes = Encoding.ASCII.GetBytes(capitalized); + var padded = new byte[Constants.CompressionNameLength]; + Array.Copy(nameBytes, padded, nameBytes.Length); + buf.Write(padded, 0, padded.Length); } buf.Write(dirIndexBytes, 0, dirIndexBytes.Length); - - foreach (var file in files) - { - file.Metadata.Write(buf); - } + foreach (var file in files) file.Metadata.Write(buf); return buf.ToArray(); } - } } diff --git a/UnrealReZen/Core/FIoStructs.cs b/UnrealReZen/Core/FIoStructs.cs index a20df74..0b7a9bb 100644 --- a/UnrealReZen/Core/FIoStructs.cs +++ b/UnrealReZen/Core/FIoStructs.cs @@ -92,8 +92,8 @@ public void AddFile(string[] fpathSections, uint fIndex, DirIndexWrapper structu } else { - FIoDirectoryIndexEntry dentry = structure.Dirs[(int)FirstChildEntry]; - FIoDirectoryIndexEntry lastDentry = null; + FIoDirectoryIndexEntry? dentry = structure.Dirs[(int)FirstChildEntry]; + FIoDirectoryIndexEntry? lastDentry = null; while (dentry != null && dentry.Name != currDirNameIndex) { @@ -160,11 +160,6 @@ public void Write(MemoryStream br) } } - public class FIoPerfectHashSeeds(uint hashSeed) - { - public uint HashSeed { get; set; } = hashSeed; - } - public class FIoChunkID(ulong id, ushort index, byte padding, byte type) { public ulong ID { get; set; } = id; @@ -200,10 +195,10 @@ public static FIoChunkID FromHexString(string hexString) public class DirIndexWrapper { - public List Dirs { get; set; } - public List Files { get; set; } - public Dictionary StrTable { get; set; } - public string[] StrSlice { get; set; } + public List Dirs { get; } + public List Files { get; } + public Dictionary StrTable { get; } + public string[] StrSlice { get; } public DirIndexWrapper(List dirs, List files, Dictionary strTable, string[] strSlice) { @@ -212,43 +207,33 @@ public DirIndexWrapper(List dirs, List gameFileMetaDatas, string mountPoint, List supportedCompressionMethods) - { - public Stream TocStream; - public byte[] AESKey; - public string UCasPath; - public UTocHeader HeaderTable = uTocHeader; - public List ChunkIdsTable; - public List OffsetAndLengthsTable; - public List PerfectHashSeedsTable; - public List CompressionBlocksDataTable; - public List SupportedCompressionMethods = supportedCompressionMethods; - public byte[] SigBlock; - public string MountPoint = mountPoint; - public List OrderedPaths; - public List ChunksMeta; - public List Files = gameFileMetaDatas; - public byte[] aesKey; - } - - public class UTocHeader - { - public string Magic; - public byte Version; - public byte[] Reserved0 = new byte[3]; - public uint HeaderSize; - public uint EntryCount; - public uint CompressedBlockEntryCount; - public uint CompressedBlockEntrySize; - public uint CompressionMethodNameCount; - public uint CompressionMethodNameLength; - public uint CompressionBlockSize; - public uint DirectoryIndexSize; - public uint PartitionCount; - public FIoContainerID ContainerID; - public FGuid EncryptionKeyGuid; - public EIoContainerFlags ContainerFlags; - public byte[] Reserved1 = new byte[3]; - public uint TocChunkPerfectHashSeedsCount; - public ulong PartitionSize; - public uint TocChunksWithoutPerfectHashCount; - public byte[] Reserved2 = new byte[44]; - - public void Write(MemoryStream br) - { - br.Write(Magic, Encoding.UTF8); - br.Write(Version); - br.Write(Reserved0); - br.Write(HeaderSize); - br.Write(EntryCount); - br.Write(CompressedBlockEntryCount); - br.Write(CompressedBlockEntrySize); - br.Write(CompressionMethodNameCount); - br.Write(CompressionMethodNameLength); - br.Write(CompressionBlockSize); - br.Write(DirectoryIndexSize); - br.Write(PartitionCount); - br.Write(ContainerID.Value); - br.Write(0); - br.Write(0); - br.Write(0); - br.Write(0); - br.Write((byte)ContainerFlags); - br.Write(Reserved1); - br.Write(TocChunkPerfectHashSeedsCount); - br.Write(PartitionSize); - br.Write(TocChunksWithoutPerfectHashCount); - br.Write(Reserved2); - - - } - - public static int SizeOf => 144; - } - - public class AssetMetadata - { - public required string FilePath; - public required FIoChunkID ChunkID; - public required FIoOffsetAndLength OffLen; - public required List CompressionBlocks; - public required FIoStoreTocEntryMeta Metadata; - } - - public class FIoContainerID(ulong value) - { - public ulong Value { get; set; } = value; - } -} - diff --git a/UnrealReZen/Core/Helpers/ChunkSource.cs b/UnrealReZen/Core/Helpers/ChunkSource.cs new file mode 100644 index 0000000..b6a6c27 --- /dev/null +++ b/UnrealReZen/Core/Helpers/ChunkSource.cs @@ -0,0 +1,58 @@ +using System.IO.MemoryMappedFiles; + +namespace UnrealReZen.Core.Helpers +{ + public sealed class ChunkSource : IDisposable + { + private readonly MemoryMappedFile? _mmf; + private readonly MemoryMappedViewAccessor? _accessor; + private readonly byte[]? _bytes; + + public long Length { get; } + + private ChunkSource(MemoryMappedFile mmf, MemoryMappedViewAccessor accessor, long length) + { + _mmf = mmf; + _accessor = accessor; + Length = length; + } + + private ChunkSource(byte[] bytes) + { + _bytes = bytes; + Length = bytes.LongLength; + } + + public static ChunkSource FromFile(string path) + { + var length = new FileInfo(path).Length; + if (length == 0) + { + return new ChunkSource(Array.Empty()); + } + var mmf = MemoryMappedFile.CreateFromFile(path, FileMode.Open, null, 0, MemoryMappedFileAccess.Read); + var accessor = mmf.CreateViewAccessor(0, length, MemoryMappedFileAccess.Read); + return new ChunkSource(mmf, accessor, length); + } + + public static ChunkSource FromBytes(byte[] bytes) => new(bytes); + + public void ReadInto(long offset, byte[] dest, int destOffset, int count) + { + if (_accessor != null) + { + _accessor.ReadArray(offset, dest, destOffset, count); + } + else + { + Array.Copy(_bytes!, offset, dest, destOffset, count); + } + } + + public void Dispose() + { + _accessor?.Dispose(); + _mmf?.Dispose(); + } + } +} diff --git a/UnrealReZen/Core/Helpers/MemoryMappedHelpers.cs b/UnrealReZen/Core/Helpers/MemoryMappedHelpers.cs deleted file mode 100644 index d6b0fa4..0000000 --- a/UnrealReZen/Core/Helpers/MemoryMappedHelpers.cs +++ /dev/null @@ -1,51 +0,0 @@ -using System.IO.MemoryMappedFiles; -using System.Security.Cryptography; - -namespace UnrealReZen.Core.Helpers -{ - public static class MemoryMappedHelpers - { - public static byte[] ReadBytesOfFile(this MemoryMappedFile mmf, long offset, long length) - { - using (MemoryMappedViewAccessor accessor = mmf.CreateViewAccessor(offset, length)) - { - byte[] data = new byte[length]; - accessor.ReadArray(0, data, 0, data.Length); - return data; - } - } - public static MemoryMappedFile CreateMemoryMappedFileFromByteArray(byte[] data, string filePath) - { - using (MemoryMappedFile mmf = MemoryMappedFile.CreateNew(Path.GetFileNameWithoutExtension(filePath), data.Length)) - { - using (MemoryMappedViewAccessor accessor = mmf.CreateViewAccessor()) - { - accessor.WriteArray(0, data, 0, data.Length); - } - return MemoryMappedFile.OpenExisting(Path.GetFileNameWithoutExtension(filePath)); - } - } - public static byte[] SHA1Hash(this MemoryMappedFile mmf) - { - using MemoryMappedViewAccessor accessor = mmf.CreateViewAccessor(); - long length = accessor.Capacity; - int bufferSize = 4096; - byte[] buffer = new byte[bufferSize]; - - using SHA1 sha1 = SHA1.Create(); - long position = 0; - while (position < length) - { - int bytesToRead = (int)Math.Min(bufferSize, length - position); - - accessor.ReadArray(position, buffer, 0, bytesToRead); - sha1.TransformBlock(buffer, 0, bytesToRead, buffer, 0); - - position += bytesToRead; - } - - sha1.TransformFinalBlock(buffer, 0, 0); - return sha1.Hash; - } - } -} diff --git a/UnrealReZen/Core/Helpers/MemoryStreamHelpers.cs b/UnrealReZen/Core/Helpers/MemoryStreamHelpers.cs index 0e7dc85..58e9ac4 100644 --- a/UnrealReZen/Core/Helpers/MemoryStreamHelpers.cs +++ b/UnrealReZen/Core/Helpers/MemoryStreamHelpers.cs @@ -1,214 +1,23 @@ -using System; -using System.Collections.Generic; -using System.IO; using System.Text; namespace UnrealReZen.Core.Helpers { - - public static class MemoryStreamOverrides + public static class MemoryStreamExtensions { + public static void Write(this MemoryStream ms, byte[] input) => ms.Write(input, 0, input.Length); - #region Write Method - //Write Method - public static void Write(this MemoryStream ms, byte[] input) - { - ms.Write(input, 0, input.Length); - } - - public static void Write(this MemoryStream ms,string input, Encoding encoding) - { - ms.Write(encoding.GetBytes(input), 0, encoding.GetBytes(input).Length); - } - - public static void Write(this MemoryStream ms,double input) - { - ms.Write(BitConverter.GetBytes(input), 0, 8); - } - - public static void Write(this MemoryStream ms,float input) - { - ms.Write(BitConverter.GetBytes(input), 0, 4); - } - - public static void Write(this MemoryStream ms,long input) - { - ms.Write(BitConverter.GetBytes(input), 0, 8); - } - - public static void Write(this MemoryStream ms,ulong input) - { - ms.Write(BitConverter.GetBytes(input), 0, 8); - } - - public static void Write(this MemoryStream ms,int input) - { - ms.Write(BitConverter.GetBytes(input), 0, 4); - } - - public static void Write(this MemoryStream ms,uint input) - { - ms.Write(BitConverter.GetBytes(input), 0, 4); - - } - - public static void Write(this MemoryStream ms,short input) - { - ms.Write(BitConverter.GetBytes(input), 0, 2); - } - - public static void Write(this MemoryStream ms,ushort input) - { - ms.Write(BitConverter.GetBytes(input), 0, 2); - - } - - public static void Write(this MemoryStream ms,byte input) - { - ms.WriteByte(input); - } - - #endregion - - #region Read Method - //Read Method - public static byte[] ReadBytes(this MemoryStream ms, int size) - { - var data = new byte[size]; - ms.Read(data, 0, size); - return data; - } - - public static byte[] ReadToEnd(this MemoryStream ms) - { - var data = new byte[ms.Length - ms.Position]; - ms.Read(data, 0, (int)(ms.Length - ms.Position)); - return data; - } - - public static byte[] ReadBytes(this MemoryStream ms, long size) - { - var data = new byte[size]; - ms.Read(data, 0, (int)size); - return data; - } - - public static short ReadInt16(this MemoryStream ms) - { - var data = new byte[2]; - ms.Read(data, 0, 2); - return BitConverter.ToInt16(data, 0); - } - - public static ushort ReadUInt16(this MemoryStream ms) - { - var data = new byte[2]; - ms.Read(data, 0, 2); - return BitConverter.ToUInt16(data, 0); - } - - public static int ReadInt32(this MemoryStream ms) - { - var data = new byte[4]; - ms.Read(data, 0, 4); - return BitConverter.ToInt32(data, 0); - } - - public static uint ReadUInt32(this MemoryStream ms) - { - var data = new byte[4]; - ms.Read(data, 0, 4); - return BitConverter.ToUInt32(data, 0); - } - - public static long ReadInt64(this MemoryStream ms) - { - var data = new byte[8]; - ms.Read(data, 0, 8); - return BitConverter.ToInt64(data, 0); - } - - public static ulong ReadUInt64(this MemoryStream ms) - { - var data = new byte[8]; - ms.Read(data, 0, 8); - return BitConverter.ToUInt64(data, 0); - } - - public static float ReadSingle(this MemoryStream ms) - { - var data = new byte[4]; - ms.Read(data, 0, 4); - return BitConverter.ToSingle(data, 0); - } - - public static string ReadString(this MemoryStream ms, int size, Encoding encoding) - { - var data = new byte[size]; - ms.Read(data, 0, size); - return encoding.GetString(data); - } - - public static string ReadString(this MemoryStream ms) - { - List output = new List(); - while (true) - { - byte reader = (byte)ms.ReadByte(); - if (reader == 0) - { - output.Add(reader); - break; - } - output.Add(reader); - } - return Encoding.ASCII.GetString(output.ToArray()); - } - - public static string ReadStringUTF8(this MemoryStream ms) - { - List output = new List(); - while (true) - { - byte reader = (byte)ms.ReadByte(); - if (reader == 0) - { - output.Add(reader); - break; - } - output.Add(reader); - } - return Encoding.UTF8.GetString(output.ToArray()); - } - #endregion - - #region Location Method - //Location Method - public static void Skip(this MemoryStream ms,int to) - { - ms.Seek(to, SeekOrigin.Current); - } - - public static void Skip(this MemoryStream ms, long to) - { - ms.Seek(to, SeekOrigin.Current); - } - - - public static long Tell(this MemoryStream ms) + public static void Write(this MemoryStream ms, string input, Encoding encoding) { - return ms.Position; + var bytes = encoding.GetBytes(input); + ms.Write(bytes, 0, bytes.Length); } - public static void Seek(this MemoryStream ms, long to) - { - ms.Seek(to, SeekOrigin.Begin); - } - - public static void Pos(this MemoryStream ms, int Base) - { - ms.Position = Base; - } - #endregion + public static void Write(this MemoryStream ms, byte input) => ms.WriteByte(input); + public static void Write(this MemoryStream ms, ushort input) => ms.Write(BitConverter.GetBytes(input), 0, 2); + public static void Write(this MemoryStream ms, short input) => ms.Write(BitConverter.GetBytes(input), 0, 2); + public static void Write(this MemoryStream ms, uint input) => ms.Write(BitConverter.GetBytes(input), 0, 4); + public static void Write(this MemoryStream ms, int input) => ms.Write(BitConverter.GetBytes(input), 0, 4); + public static void Write(this MemoryStream ms, ulong input) => ms.Write(BitConverter.GetBytes(input), 0, 8); + public static void Write(this MemoryStream ms, long input) => ms.Write(BitConverter.GetBytes(input), 0, 8); } } diff --git a/UnrealReZen/Core/UToc.cs b/UnrealReZen/Core/UToc.cs new file mode 100644 index 0000000..9702553 --- /dev/null +++ b/UnrealReZen/Core/UToc.cs @@ -0,0 +1,73 @@ +using CUE4Parse.UE4.Objects.Core.Misc; +using System.Text; +using UnrealReZen.Core.Helpers; + +namespace UnrealReZen.Core +{ + public class UTocHeader + { + public required string Magic { get; init; } + public required byte Version { get; init; } + public byte[] Reserved0 { get; init; } = new byte[3]; + public required uint HeaderSize { get; init; } + public required uint EntryCount { get; init; } + public required uint CompressedBlockEntryCount { get; init; } + public required uint CompressedBlockEntrySize { get; init; } + public required uint CompressionMethodNameCount { get; init; } + public required uint CompressionMethodNameLength { get; init; } + public required uint CompressionBlockSize { get; init; } + public required uint DirectoryIndexSize { get; init; } + public required uint PartitionCount { get; init; } + public required FIoContainerID ContainerID { get; init; } + public FGuid EncryptionKeyGuid { get; init; } + public required EIoContainerFlags ContainerFlags { get; init; } + public byte[] Reserved1 { get; init; } = new byte[3]; + public uint TocChunkPerfectHashSeedsCount { get; init; } + public required ulong PartitionSize { get; init; } + public uint TocChunksWithoutPerfectHashCount { get; init; } + public byte[] Reserved2 { get; init; } = new byte[44]; + + public const int SizeOf = 144; + + public void Write(MemoryStream ms) + { + ms.Write(Magic, Encoding.UTF8); + ms.Write(Version); + ms.Write(Reserved0); + ms.Write(HeaderSize); + ms.Write(EntryCount); + ms.Write(CompressedBlockEntryCount); + ms.Write(CompressedBlockEntrySize); + ms.Write(CompressionMethodNameCount); + ms.Write(CompressionMethodNameLength); + ms.Write(CompressionBlockSize); + ms.Write(DirectoryIndexSize); + ms.Write(PartitionCount); + ms.Write(ContainerID.Value); + ms.Write(0); + ms.Write(0); + ms.Write(0); + ms.Write(0); + ms.Write((byte)ContainerFlags); + ms.Write(Reserved1); + ms.Write(TocChunkPerfectHashSeedsCount); + ms.Write(PartitionSize); + ms.Write(TocChunksWithoutPerfectHashCount); + ms.Write(Reserved2); + } + } + + public class AssetMetadata + { + public required string FilePath { get; set; } + public required FIoChunkID ChunkID { get; init; } + public required FIoOffsetAndLength OffLen { get; init; } + public required List CompressionBlocks { get; init; } + public required FIoStoreTocEntryMeta Metadata { get; init; } + } + + public class FIoContainerID(ulong value) + { + public ulong Value { get; } = value; + } +} diff --git a/UnrealReZen/Program.cs b/UnrealReZen/Program.cs index 331eb18..0cae02d 100644 --- a/UnrealReZen/Program.cs +++ b/UnrealReZen/Program.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using CommandLine.Text; using CUE4Parse.Compression; using CUE4Parse.Encryption.Aes; @@ -14,7 +14,7 @@ namespace UnrealReZen { - class Options + internal class Options { [Option('g', "game-dir", Required = true, HelpText = "Path to the game directory (for loading UCAS and UTOC files).")] public required string GameDirectory { get; set; } @@ -28,14 +28,17 @@ class Options [Option('o', "output-path", Required = true, HelpText = "Path (including file name) for the packed utoc file.")] public required string OutputPath { get; set; } - [Option('a', "aes-key", Required = false, HelpText = "AES key of the game (only if its encrypted)")] + [Option('a', "aes-key", Required = false, HelpText = "AES key for reading the game's encrypted source archives. Not used to encrypt the output unless --encrypt-output is also set.")] public string? AESKey { get; set; } + [Option("encrypt-output", Required = false, Default = false, HelpText = "Encrypt the generated .ucas with the game's AES key and set the EncryptedContainerFlag in the .utoc. Most games accept plain archives for mods; leave off unless you know the target refuses unencrypted containers.")] + public bool EncryptOutput { get; set; } + [Option("compression-format", Required = false, Default = "Zlib", HelpText = "Compression format (None, Zlib, Oodle, LZ4).")] - public string CompressionFormat { get; set; } + public string CompressionFormat { get; set; } = "Zlib"; [Option("mount-point", Required = false, Default = "../../../", HelpText = "Mount point of packed archive")] - public string MountPoint { get; set; } + public string MountPoint { get; set; } = "../../../"; [Option("container-id", Required = false, HelpText = "Container Id of packed archive (default is a random 8-byte number)")] public ulong? ContainerId { get; set; } @@ -44,141 +47,234 @@ class Options public bool GameDirTopOnly { get; set; } [Usage(ApplicationAlias = "UnrealReZen.exe")] - public static IEnumerable Examples - { - get + public static IEnumerable Examples => [ + new("Making a patch for a ue5 game", new Options { - return [ - new("Making a patch for a ue5 game", new Options { GameDirectory = "C:/Games/MyGame",ContentPath = "C:/Games/MyGame/ExportedFiles", EngineVersion = "GAME_UE5_1", CompressionFormat = "Zlib", OutputPath = "C:/Games/MyGame/TestPatch_P.utoc"}) - ]; - } - } - + GameDirectory = "C:/Games/MyGame", + ContentPath = "C:/Games/MyGame/ExportedFiles", + EngineVersion = "GAME_UE5_1", + CompressionFormat = "Zlib", + OutputPath = "C:/Games/MyGame/TestPatch_P.utoc" + }) + ]; } - internal class Program + + internal static class Program { - static void Main(string[] args) + private const int ExitOk = 0; + private const int ExitError = 1; + + static int Main(string[] args) { Log.Logger = new LoggerConfiguration() - .MinimumLevel.Debug() - .WriteTo.Console() - .CreateLogger(); + .MinimumLevel.Debug() + .WriteTo.Console() + .CreateLogger(); + int exitCode = ExitError; Parser.Default.ParseArguments(args) - .WithParsed(RunOptionsAndReturnExitCode); + .WithParsed(opts => exitCode = Run(opts)); + return exitCode; } - static void RunOptionsAndReturnExitCode(Options opts) + private static int Run(Options opts) { Constants.ToolDirectory = AppDomain.CurrentDomain.BaseDirectory; - if (!OodleHelper.DownloadOodleDll(Path.Combine(Constants.ToolDirectory, OodleHelper.OODLE_DLL_NAME))) + + if (!EnsureNativeDlls()) return ExitError; + if (!TryParseEngineVersion(opts.EngineVersion, out var engineVersion)) return ExitError; + if (!ValidateCliOptions(opts)) return ExitError; + + LogOptionsSummary(opts, engineVersion); + + var aesKey = new FAesKey(opts.AESKey ?? Constants.DefaultAES); + if (!TryLoadProvider(opts, engineVersion, aesKey, out var provider)) return ExitError; + + foreach (var vfs in provider.MountedVfs) + { + vfs.Dispose(); + } + + var filesToRepack = Directory.GetFiles(opts.ContentPath, "*", SearchOption.AllDirectories); + if (filesToRepack.Length == 0) + { + Log.Fatal("No valid files found in the content path"); + return ExitError; + } + + Log.Information("Packing Contents..."); + var manifest = BuildManifest(provider, opts, engineVersion, filesToRepack); + + Log.Information("Packing files..."); + var outputAesKey = opts.EncryptOutput ? aesKey : null; + Packer.PackToCasToc(opts.ContentPath, manifest, opts.OutputPath, opts.CompressionFormat, outputAesKey, opts.MountPoint, engineVersion); + Console.WriteLine($"Done! {filesToRepack.Length} file(s) packed"); + return ExitOk; + } + + private static bool EnsureNativeDlls() + { + try + { + OodleHelper.Initialize(Path.Combine(Constants.ToolDirectory, OodleHelper.OodleFileName)); + } + catch (Exception ex) + { + Log.Fatal(ex, $"Failed to initialize Oodle. Place {OodleHelper.OodleFileName} in the tool directory and try again."); + return false; + } + if (OodleHelper.Instance is null) + { + Log.Fatal($"Oodle was not registered. Place {OodleHelper.OodleFileName} in the tool directory or check your internet connection."); + return false; + } + + try { - Log.Fatal("UnrealReZen failed to download the oodle dll. please check you internet connection or place oo2core_9_win64.dll in the tool directory"); - Console.ReadLine(); - return; + ZlibHelper.Initialize(Path.Combine(Constants.ToolDirectory, ZlibHelper.DllName)); } - if (!ZlibHelper.DownloadDll(Path.Combine(Constants.ToolDirectory, ZlibHelper.DLL_NAME))) + catch (Exception ex) + { + Log.Fatal(ex, $"Failed to initialize Zlib. Place {ZlibHelper.DllName} in the tool directory and try again."); + return false; + } + if (ZlibHelper.Instance is null) { - Log.Fatal("UnrealReZen failed to download the zlib dll. please check you internet connection or place zlib-ng2.dll in the tool directory"); - Console.ReadLine(); - return; + Log.Fatal($"Zlib was not registered. Place {ZlibHelper.DllName} in the tool directory or check your internet connection."); + return false; } - if (!Enum.TryParse(typeof(EGame), opts.EngineVersion, out var engineVersion)) + return true; + } + + private static bool TryParseEngineVersion(string raw, out EGame engineVersion) + { + if (Enum.TryParse(raw, out engineVersion) && Enum.IsDefined(engineVersion)) { - Log.Fatal("Invalid Unreal Engine version. Please enter a valid version (e.g., GAME_UE4_0)."); - Log.Information("List of supported engine versions: " + string.Join("\n", Enum.GetNames(typeof(EGame)))); - return; + return true; } - if (!Constants.CompressionTypes.Contains(opts.CompressionFormat.ToLower())) + Log.Fatal("Invalid Unreal Engine version. Please enter a valid version (e.g., GAME_UE4_0)."); + Log.Information("List of supported engine versions: " + string.Join("\n", Enum.GetNames())); + return false; + } + + private static bool ValidateCliOptions(Options opts) + { + if (!Constants.CompressionTypes.Contains(opts.CompressionFormat.ToLowerInvariant())) { Log.Fatal($"Unsupported compression format : {opts.CompressionFormat}"); - return; + return false; } - if (Path.GetExtension(opts.OutputPath) != ".utoc") + if (!string.Equals(Path.GetExtension(opts.OutputPath), ".utoc", StringComparison.OrdinalIgnoreCase)) { - Log.Fatal($"Output path must contains utoc extension"); - return; + Log.Fatal("Output path must contain utoc extension"); + return false; } + return true; + } + private static void LogOptionsSummary(Options opts, EGame engineVersion) + { Console.WriteLine($"Game Directory: {opts.GameDirectory}"); Console.WriteLine($"Content Path: {opts.ContentPath}"); Console.WriteLine($"Unreal Engine Version: {engineVersion}"); Console.WriteLine($"Output Path: {opts.OutputPath}"); + } - var aesKey = new FAesKey(opts.AESKey ?? Constants.DefaultAES); - DefaultFileProvider provider; - + private static bool TryLoadProvider(Options opts, EGame engineVersion, FAesKey aesKey, out DefaultFileProvider provider) + { Log.Information("Loading Game Archives..."); try { var searchOption = opts.GameDirTopOnly ? SearchOption.TopDirectoryOnly : SearchOption.AllDirectories; - provider = new DefaultFileProvider(opts.GameDirectory, searchOption, true, new VersionContainer((EGame)engineVersion)); + provider = new DefaultFileProvider(opts.GameDirectory, searchOption, new VersionContainer(engineVersion), StringComparer.OrdinalIgnoreCase); provider.Initialize(); provider.SubmitKey(new FGuid(), aesKey); provider.LoadLocalization(ELanguage.English); if (provider.RequiredKeys.Count > 0 && provider.Keys.Count == 0) { - Log.Fatal("Some of archives needs AES Key. Please enter aes key"); - return; + Log.Fatal("Some archives require an AES key. Please provide --aes-key."); + return false; } + return true; } catch (Exception ex) { - Log.Fatal("Error:" + ex.ToString()); + Log.Fatal("Error: " + ex); Log.Information("Maybe changing aes key or engine version helps"); - return; + provider = null!; + return false; } + } - foreach (var vfs in provider.MountedVfs) + private static Dependency BuildManifest(DefaultFileProvider provider, Options opts, EGame engineVersion, string[] filesToRepack) + { + var newContainerId = opts.ContainerId ?? CryptographyHelpers.RandomUlong(); + byte containerChunkType = engineVersion >= EGame.GAME_UE5_0 + ? (byte)EIoChunkType5.ContainerHeader + : (byte)EIoChunkType.ContainerHeader; + + var manifest = new Dependency { - vfs.Dispose(); - } + Deps = new DependenciesData + { + ThisPackageID = newContainerId, + ChunkIDToDependencies = [] + }, + Files = [new ManifestFile + { + ChunkID = new FIoChunkID(newContainerId, 0, 0, containerChunkType), + Filepath = Constants.DepFileName + }] + }; - Log.Information("Packing Contents..."); - Dependency m = new() { Deps = new DependenciesData { ChunkIDToDependencies = [] }, Files = [] }; - List FilesToRepack = new(Directory.GetFiles(opts.ContentPath, "*", SearchOption.AllDirectories)); - var newContainerID = opts.ContainerId ?? CryptographyHelpers.RandomUlong(); + var utocEntryLookup = BuildUtocEntryLookup(provider); - byte type = (EGame)engineVersion >= EGame.GAME_UE5_0 ? (byte)EIoChunkType5.ContainerHeader : (byte)EIoChunkType.ContainerHeader; - m.Files.Add(new ManifestFile { ChunkID = new FIoChunkID(newContainerID, 0, 0, type), Filepath = Constants.DepFileName }); - m.Deps.ThisPackageID = newContainerID; - if (FilesToRepack.Count == 0) + foreach (var file in filesToRepack) { - Log.Fatal("No valid files found in the content path"); - return; - } - foreach (var file in FilesToRepack) - { - string filename = file.Replace(opts.ContentPath + "\\", "").Replace("\\", "/"); + string filename = Path.GetRelativePath(opts.ContentPath, file).Replace('\\', '/'); Log.Information("Mounting " + Path.GetFileName(filename)); - var filedata = provider.Files.Values.Where(a => Path.GetExtension(((AbstractVfsReader)((VfsEntry)a).Vfs).Name) == ".utoc" && a.Path.Equals(filename, StringComparison.CurrentCultureIgnoreCase)); - if (filedata == null || !filedata.Any()) + if (!utocEntryLookup.TryGetValue(filename, out var matches)) { - Log.Warning("Skipping " + filename + " because its not found in archives."); + Log.Warning("Skipping " + filename + " because it's not found in archives."); continue; } - foreach (var dep in filedata) + foreach (var entry in matches) { - dynamic b = dep; - FIoChunkId c = b.ChunkId; - m.Files.Add(new ManifestFile { Filepath = dep.Path, ChunkID = new FIoChunkID(c.ChunkId, 0, 0, c.ChunkType) }); - IoStoreReader IoFile = b.IoStoreReader; - if (IoFile.ContainerHeader != null) + var chunkId = entry.ChunkId; + manifest.Files.Add(new ManifestFile + { + Filepath = entry.Path, + ChunkID = new FIoChunkID(chunkId.ChunkId, 0, 0, chunkId.ChunkType) + }); + + var header = entry.IoStoreReader.ContainerHeader; + if (header != null) { - foreach (var st in IoFile.ContainerHeader.StoreEntries) + foreach (var storeEntry in header.StoreEntries) { - if (m.Deps.ChunkIDToDependencies.ContainsKey(c.ChunkId)) continue; - m.Deps.ChunkIDToDependencies.Add(c.ChunkId, st); + manifest.Deps.ChunkIDToDependencies.TryAdd(chunkId.ChunkId, storeEntry); } } } } - Log.Information("Packing files..."); - Packer.PackToCasToc(opts.ContentPath, m, opts.OutputPath, opts.CompressionFormat, aesKey, opts.MountPoint, (EGame)engineVersion); - Console.WriteLine($"Done! {FilesToRepack.Count} file(s) packed"); + return manifest; + } + private static Dictionary> BuildUtocEntryLookup(DefaultFileProvider provider) + { + var lookup = new Dictionary>(StringComparer.OrdinalIgnoreCase); + foreach (var entry in provider.Files.Values.OfType()) + { + if (Path.GetExtension(((AbstractVfsReader)entry.Vfs).Name) != ".utoc") continue; + if (!lookup.TryGetValue(entry.Path, out var list)) + { + list = new List(); + lookup[entry.Path] = list; + } + list.Add(entry); + } + return lookup; } } - } diff --git a/UnrealReZen/UnrealReZen.csproj b/UnrealReZen/UnrealReZen.csproj index a11087b..21ea3b4 100644 --- a/UnrealReZen/UnrealReZen.csproj +++ b/UnrealReZen/UnrealReZen.csproj @@ -11,10 +11,13 @@ - + + + + True diff --git a/external/CUE4Parse b/external/CUE4Parse new file mode 160000 index 0000000..9539d4d --- /dev/null +++ b/external/CUE4Parse @@ -0,0 +1 @@ +Subproject commit 9539d4d86aef568f27957796249891d125d2c223