Merge branch 'dev'

.editorconfig

@@ -117,3 +117,6 @@ indent_size = 4
 indent_style = space
 indent_size = 4
 tab_width = 4
+
+[*.md]
+trim_trailing_whitespace = false

NexusMods.Archives.Nx.Benchmarks/Benchmarks/PackArchiveToDisk.cs

@@ -7,7 +7,7 @@ namespace NexusMods.Archives.Nx.Benchmarks.Benchmarks;
 
 public class PackArchiveToDisk
 {
-    public string Directory { get; set; } = @"C:/Users/sewer/Desktop/Sonic Heroes";
+    public string Directory { get; set; } = @"/home/sewer/Games/SonicHeroes";
     public string OutputPath { get; set; } = @"PackArchiveToDisk.nx";
     public PackerFile[] Files { get; set; } = null!;
 

NexusMods.Archives.Nx.Benchmarks/Benchmarks/ParsingTableOfContents.cs

@@ -26,6 +26,7 @@ public class ParsingTableOfContents
     internal TableOfContentsBuilder<PackerFileForBenchmarking> Builder { get; set; } = null!;
     internal byte[] PrebuiltData { get; set; } = null!;
     internal PackerFileForBenchmarking[] Files { get; set; } = null!;
+
     internal List<IBlock<PackerFileForBenchmarking>> Blocks { get; set; } = null!;
     public Dictionary<string, List<PackerFileForBenchmarking>> Groups { get; set; } = null!;
 
@@ -34,7 +35,8 @@ public void Setup()
     {
         var entries = Assets.GetYakuzaFileEntries();
         Files = entries.Take(N).Select(x => new PackerFileForBenchmarking(x.RelativePath, x.FileSize)).ToArray();
-        CreateToc();
+        CreateToc(out var blocks);
+        Blocks = blocks;
     }
 
     // Size Reporting
@@ -46,12 +48,16 @@ public void Cleanup_CreateTable()
     }
 
     [Benchmark]
-    public int CreateTable() => CreateToc();
+    public int CreateTable() => CreateToc(out _);
 
-    //[Benchmark]
-    public void InitTableData() => InitTocData();
+    [Benchmark]
+    public int InitTableData()
+    {
+        var data = InitTocData();
+        return data.Count; // prevent dead code elimination.
+    }
 
-    //[Benchmark]
+    [Benchmark]
     public void InitTable()
     {
         using var toc = InitToc();
@@ -62,29 +68,29 @@ public unsafe TableOfContents ParseTable()
     {
         fixed (byte* dataPtr = PrebuiltData)
         {
-            return TableOfContents.Deserialize<TableOfContents>(dataPtr, 0, PrebuiltData.Length, Builder.Version);
+            return TableOfContents.Deserialize<TableOfContents>(dataPtr);
         }
     }
 
-    private TableOfContentsBuilder<PackerFileForBenchmarking> InitToc() => new(Blocks, Files);
+    private TableOfContentsBuilder<PackerFileForBenchmarking> InitToc() => TableOfContentsBuilder<PackerFileForBenchmarking>.Create<PackerFileForBenchmarking>(Blocks, Files);
 
-    private void InitTocData()
+    private List<IBlock<PackerFileForBenchmarking>> InitTocData()
     {
         // Generate blocks.
         Groups = GroupFiles.Do(Files);
-        Blocks = MakeBlocks.Do(Groups, SolidBlockSize, ChunkSize);
+        return MakeBlocks.Do(Groups, SolidBlockSize, ChunkSize);
     }
 
-    private unsafe int CreateToc()
+    private unsafe int CreateToc(out List<IBlock<PackerFileForBenchmarking>> blocks)
     {
         Builder.Dispose();
 
         // Generate blocks.
         Groups = GroupFiles.Do(Files);
-        Blocks = MakeBlocks.Do(Groups, SolidBlockSize, ChunkSize);
+        blocks = MakeBlocks.Do(Groups, SolidBlockSize, ChunkSize);
 
         // Generate TOC.
-        Builder = new TableOfContentsBuilder<PackerFileForBenchmarking>(Blocks, Files);
+        Builder = TableOfContentsBuilder<PackerFileForBenchmarking>.Create<PackerFileForBenchmarking>(blocks, Files);
         foreach (var unused in Files)
         {
             ref var _ = ref Builder.GetAndIncrementFileAtomic();
@@ -95,7 +101,7 @@ private unsafe int CreateToc()
         PrebuiltData = new byte[tocSize];
         fixed (byte* dataPtr = PrebuiltData)
         {
-            return Builder.Build(dataPtr, 0, tocSize);
+            return Builder.Build(dataPtr, tocSize);
         }
     }
 }

NexusMods.Archives.Nx.Benchmarks/NexusMods.Archives.Nx.Benchmarks.csproj

@@ -2,11 +2,11 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFramework>net7.0</TargetFramework>
+        <TargetFramework>net9.0</TargetFramework>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
     </PropertyGroup>
-    
+
     <ItemGroup>
         <PackageReference Include="BenchmarkDotNet" Version="0.13.5" />
     </ItemGroup>

NexusMods.Archives.Nx.Benchmarks/Program.cs

@@ -16,6 +16,6 @@
 //BenchmarkRunner.Run<OutputWriteLock>();
 
 // Modify path in source code before executing these ones below //
-//BenchmarkRunner.Run<PackArchiveToDisk>();
+// BenchmarkRunner.Run<PackArchiveToDisk>();
 //BenchmarkRunner.Run<UnpackArchiveFromDisk>();
 Console.WriteLine("Hello World 😺 !!");

NexusMods.Archives.Nx.Cli/NexusMods.Archives.Nx.Cli.csproj

@@ -2,20 +2,20 @@
 
     <PropertyGroup>
         <OutputType>Exe</OutputType>
-        <TargetFrameworks>net7.0;net8.0</TargetFrameworks>
+        <TargetFrameworks>net7.0;net8.0;net9.0</TargetFrameworks>
         <ImplicitUsings>enable</ImplicitUsings>
         <Nullable>enable</Nullable>
-        
+
         <!-- Enable Dynamic PGO for targets lower than net8.0 -->
         <TieredPGO>enable</TieredPGO>
-        
+
         <!-- Native Binaries go Brrrrr -->
         <PublishAot>true</PublishAot>
 
         <!-- For Analyzing Binary Output Size with sizoscope -->
         <IlcGenerateMstatFile>true</IlcGenerateMstatFile>
         <IlcGenerateDgmlFile>true</IlcGenerateDgmlFile>
-        
+
         <!-- Optimize NativeAOT -->
         <OptimizationPreference>Speed</OptimizationPreference>
         <EnableUnsafeBinaryFormatterSerialization>false</EnableUnsafeBinaryFormatterSerialization>

NexusMods.Archives.Nx.Cli/Program.cs

@@ -5,6 +5,7 @@
 using System.Diagnostics;
 using NexusMods.Archives.Nx.Enums;
 using NexusMods.Archives.Nx.FileProviders;
+using NexusMods.Archives.Nx.Headers;
 using NexusMods.Archives.Nx.Packing;
 using NexusMods.Archives.Nx.Structs;
 using Spectre.Console;
@@ -45,17 +46,32 @@
     new Option<int?>("--chunkedlevel", () => defaultPackerSettings.ChunkedCompressionLevel, "Compression level to use for chunks of large data. ZStandard has Range -5 - 22. LZ4 has Range: 1 - 12."),
     new Option<CompressionPreference?>("--solid-algorithm", () => defaultPackerSettings.SolidBlockAlgorithm, "Compression algorithm used for compressing SOLID blocks."),
     new Option<CompressionPreference?>("--chunked-algorithm", () => defaultPackerSettings.ChunkedFileAlgorithm, "Compression algorithm used for compressing chunked files."),
+    new Option<bool>("--deduplicate-chunked", () => false, "Enable CHUNKED block file deduplication during packing."),
+    new Option<bool>("--deduplicate-solid", () => true, "Enable SOLID block file deduplication during packing."),
     maxNumThreads
 };
 
-packCommand.Handler = CommandHandler.Create<string, string, int?, int?, int?, int?, CompressionPreference?, CompressionPreference?, int?>(Pack);
+packCommand.Handler = CommandHandler.Create<string, string, int?, int?, int?, int?, CompressionPreference?, CompressionPreference?, int?, bool, bool>(Pack);
+
+// Merge Command
+var mergeCommand = new Command("merge", "Merge multiple .nx archives into a single archive")
+{
+    new Option<string[]>("--sources", "Source .nx archives to merge.") { IsRequired = true, AllowMultipleArgumentsPerToken = true },
+    new Option<string>("--output", "Output path for the merged archive.") { IsRequired = true },
+    new Option<bool>("--deduplicate-chunked", () => false, "Enable CHUNKED block file deduplication during merging."),
+    new Option<bool>("--deduplicate-solid", () => true, "Enable SOLID block file deduplication during merging."),
+    maxNumThreads
+};
+
+mergeCommand.Handler = CommandHandler.Create<string[], string, int?, bool, bool>(Merge);
 
 // Root command
 var rootCommand = new RootCommand
 {
     extractCommand,
     packCommand,
-    benchmarkCommand
+    benchmarkCommand,
+    mergeCommand
 };
 
 // Parse the incoming args and invoke the handler
@@ -90,9 +106,9 @@ void Extract(string source, string target, int? threads)
     Console.WriteLine("Unpacked in {0}ms", unpackingTimeTaken.ElapsedMilliseconds);
 }
 
-void Pack(string source, string target, int? blocksize, int? chunksize, int? solidLevel, int? chunkedLevel, CompressionPreference? solidAlgorithm, CompressionPreference? chunkedAlgorithm, int? threads)
+void Pack(string source, string target, int? blocksize, int? chunksize, int? solidLevel, int? chunkedLevel, CompressionPreference? solidAlgorithm, CompressionPreference? chunkedAlgorithm, int? threads, bool deduplicateChunked, bool deduplicateSolid)
 {
-    Console.WriteLine($"Packing {source} to {target} with {threads} threads, blocksize [{blocksize}], chunksize [{chunksize}], solidLevel [{solidLevel}], chunkedLevel [{chunkedLevel}], solidAlgorithm [{solidAlgorithm}], chunkedAlgorithm [{chunkedAlgorithm}].");
+    Console.WriteLine($"Packing {source} to {target} with {threads} threads, blocksize [{blocksize}], chunksize [{chunksize}], solidLevel [{solidLevel}], chunkedLevel [{chunkedLevel}], solidAlgorithm [{solidAlgorithm}], chunkedAlgorithm [{chunkedAlgorithm}], deduplicate [chunked: {deduplicateChunked} solid: {deduplicateSolid}].");
 
     var builder = new NxPackerBuilder();
     builder.AddFolder(source);
@@ -119,6 +135,9 @@ void Pack(string source, string target, int? blocksize, int? chunksize, int? sol
     if (threads.HasValue)
         builder.WithMaxNumThreads(threads.Value);
 
+    builder.WithChunkedDeduplication(deduplicateChunked);
+    builder.WithSolidDeduplication(deduplicateSolid);
+
     var packingTimeTaken = Stopwatch.StartNew();
 
     // Progress Reporting.
@@ -135,7 +154,7 @@ void Pack(string source, string target, int? blocksize, int? chunksize, int? sol
     var ms = packingTimeTaken.ElapsedMilliseconds;
     Console.WriteLine("Packed in {0}ms", ms);
     Console.WriteLine("Throughput {0:###.00}MiB/s", builder.Files.Sum(x => x.FileSize) / (float)ms / 1024F);
-    Console.WriteLine("Size {0} Bytes", builder.Settings.Output.Length);
+    Console.WriteLine("Size {0:F2} MiB", BytesToMiB((ulong)builder.Settings.Output.Length));
     builder.Settings.Output.Dispose();
 }
 
@@ -171,3 +190,55 @@ void Benchmark(string source, int? threads, int? attempts)
     Console.WriteLine("Average {0:###.00}ms", averageMs);
     Console.WriteLine("Throughput {0:###.00}GiB/s", outputs.Sum(x => (long)x.Data.Length) / averageMs / 1048576F);
 }
+
+void Merge(string[] sources, string output, int? threads, bool deduplicateChunked, bool deduplicateSolid)
+{
+    Console.WriteLine($"Merging {sources.Length} archives into {output} with [{threads}] threads [deduplicate-chunked: {deduplicateChunked}, deduplicate-solid: {deduplicateSolid}].");
+
+    var builder = new NxDeduplicatingRepackerBuilder();
+    ulong totalInputSize = 0;
+    ulong totalDecompressedSize = 0;
+
+    if (threads.HasValue)
+        builder.WithMaxNumThreads(threads.Value);
+
+    builder.WithChunkedDeduplication(deduplicateChunked);
+    builder.WithSolidDeduplication(deduplicateSolid);
+
+    foreach (var source in sources)
+    {
+        var provider = new FromFilePathProvider { FilePath = source };
+        var fileInfo = new FileInfo(source);
+
+        using var fileData = provider.GetFileData(0, (ulong)fileInfo.Length);
+        totalInputSize += fileData.DataLength;
+        var header = HeaderParser.ParseHeader(provider);
+        builder.AddFilesFromNxArchive(provider, header, header.Entries.AsSpan());
+
+        foreach (var entry in header.Entries)
+            totalDecompressedSize += entry.DecompressedSize;
+    }
+
+    var mergeTimeTaken = Stopwatch.StartNew();
+
+    // Progress Reporting.
+    AnsiConsole.Progress()
+        .Start(ctx =>
+        {
+            var mergeTask = ctx.AddTask("[green]Merging Archives[/]");
+            var progress = new Progress<double>(d => mergeTask.Value = d * 100);
+            builder.WithProgress(progress);
+            builder.WithOutput(File.Create(output));
+            builder.Build(false);
+        });
+
+    var outputSize = builder.Settings.Output.Length;
+    var ms = mergeTimeTaken.ElapsedMilliseconds;
+    Console.WriteLine("Merged in {0}ms", ms);
+    Console.WriteLine("Input Size: {0:F2} MiB", BytesToMiB(totalInputSize));
+    Console.WriteLine("Output Size: {0:F2} MiB", BytesToMiB((ulong)outputSize));
+    Console.WriteLine("Compression Ratio: {0:P2}", (double)outputSize / totalInputSize);
+    Console.WriteLine("Throughput {0:###.00}MiB/s", totalDecompressedSize / (float)ms / 1024F);
+}
+
+static float BytesToMiB(ulong bytes) => bytes / 1024F / 1024F;

NexusMods.Archives.Nx.Tests/Assets.cs

@@ -0,0 +1,49 @@
+namespace NexusMods.Archives.Nx.Tests;
+
+/// <summary>
+///     Provides access to test assets.
+/// </summary>
+public static class Assets
+{
+    private static string TestRunnerDirectory => AppContext.BaseDirectory;
+
+    /// <summary>
+    ///     This folder stores all tests for repacking .nx archives.
+    /// </summary>
+    public static class Repacks
+    {
+        public static class Unit
+        {
+            /// <summary>
+            ///     We've changed a file in a SOLID block. The block should be repacked.
+            /// </summary>
+            public static class ChangeInSolidBlock
+            {
+                public static string New => Path.Combine(TestRunnerDirectory, "Assets/Repacks/Unit/Change-In-Solid-Block/New");
+                public static string Original => Path.Combine(TestRunnerDirectory, "Assets/Repacks/Unit/Change-In-Solid-Block/Original");
+            }
+
+            /// <summary>
+            ///     A chunked file is left unchanged. It should be reused.
+            ///     Run this test with chunk size of 32K.
+            /// </summary>
+            public static class ChunkedFileUnchanged
+            {
+                public static string Original => Path.Combine(TestRunnerDirectory, "Assets/Repacks/Unit/Chunked-File-Unchanged/Original");
+            }
+        }
+
+        public static class Integration
+        {
+            /// <summary>
+            ///     We've changed a file in a SOLID block. The block should be repacked.
+            ///     But there's also another block that's unchanged.
+            /// </summary>
+            public static class ChangeInSolidBlockWithAnotherUnchangedBlock
+            {
+                public static string New => Path.Combine(TestRunnerDirectory, "Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/New");
+                public static string Original => Path.Combine(TestRunnerDirectory, "Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/Original");
+            }
+        }
+    }
+}

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/New/block0-haschange/0-unchanged.txt

@@ -0,0 +1,2 @@
+This is the first file in block 0.
+This file should not change between updates.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/New/block0-haschange/1-unchanged.txt

@@ -0,0 +1,2 @@
+This is the second file in block 0.
+This file should not change between updates.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/New/block0-haschange/2-changed.txt

@@ -0,0 +1,3 @@
+This is the third file in block 0.
+
+This file HAS CHANGED after update.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/New/block1-unchanged/0.txt2

@@ -0,0 +1,4 @@
+This file represents another block.
+
+It uses `txt2` extension because Nx groups files by extension by default,
+so this makes it easier to test, as files don't have to be grouped manually.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/New/block1-unchanged/1.txt2

@@ -0,0 +1 @@
+This is just another file to be included in this block.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/Original/block0-haschange/0-unchanged.txt

@@ -0,0 +1,2 @@
+This is the first file in block 0.
+This file should not change between updates.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/Original/block0-haschange/1-unchanged.txt

@@ -0,0 +1,2 @@
+This is the second file in block 0.
+This file should not change between updates.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/Original/block0-haschange/2-changed.txt

@@ -0,0 +1,3 @@
+This is the third file in block 0.
+
+This file WILL CHANGE after update.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/Original/block1-unchanged/0.txt2

@@ -0,0 +1,4 @@
+This file represents another block.
+
+It uses `txt2` extension because Nx groups files by extension by default,
+so this makes it easier to test, as files don't have to be grouped manually.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Integration/Change-In-Solid-Block-With-Another-Unchanged-Block/Original/block1-unchanged/1.txt2

@@ -0,0 +1 @@
+This is just another file to be included in this block.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Unit/Change-In-Solid-Block/New/0-unchanged.txt

@@ -0,0 +1,2 @@
+This is the first file in block 0.
+This file should not change between updates.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Unit/Change-In-Solid-Block/New/1-unchanged.txt

@@ -0,0 +1,2 @@
+This is the second file in block 0.
+This file should not change between updates.

NexusMods.Archives.Nx.Tests/Assets/Repacks/Unit/Change-In-Solid-Block/New/2-changed.txt

@@ -0,0 +1,3 @@
+This is the third file in block 0.
+
+This file HAS CHANGED after update.