Merge pull request #20 from PikminGuts92/dev

Optimizations + Wii RB2 texture decoding

Src/Apps/ArkHelper/Apps/Dir2ArkApp.cs

@@ -156,7 +156,7 @@ public void Parse(Dir2ArkOptions op)
             else if ((int)ark.Version >= 3 && potentialPartSize >= arkPartSizeLimit)
             {
                 // Kind of hacky but multiple part writing isn't implemented in commit changes yet
-                ark.CommitChanges(true);
+                ark.CommitChanges(false);
                 ark.AddAdditionalPart();
 
                 currentPartSize = 0;
@@ -176,7 +176,8 @@ public void Parse(Dir2ArkOptions op)
             currentPartSize += fileSize;
         }
 
-        ark.CommitChanges(true);
+        ark.CommitChanges(false);
+        ark.WriteHeader();
         watch.Stop();
 
         Log.Information("Finished building ark in {WatchElapsed}", watch.Elapsed);

Src/Apps/P9SongTool/Apps/Milo2ProjectApp.cs

@@ -114,10 +114,10 @@ protected SystemInfo GetSystemInfo(Milo2ProjectOptions op)
         {
             Version = 25,
             BigEndian = true,
-            Platform = op.InputPath.ToLower() switch
+            Platform = op.InputPath switch
             {
-                var p when p.EndsWith("_ps3") => Platform.PS3,
-                var p when p.EndsWith("_wii") => Platform.Wii,
+                var p when p.EndsWith("_ps3", StringComparison.InvariantCultureIgnoreCase) => Platform.PS3,
+                var p when p.EndsWith("_wii", StringComparison.InvariantCultureIgnoreCase) => Platform.Wii,
                 _ => Platform.X360
             }
         };

Src/Core/Mackiloha.App/Extensions/TextureExtensions.cs

@@ -45,16 +45,47 @@ public static byte[] ToRGBA(this HMXBitmap bitmap, SystemInfo info)
             case TPL_CMP_2:
             case TPL_CMP_ALPHA:
                 // Wii textures
-                var tempData = new byte[((bitmap.Width * bitmap.Height) * bitmap.Bpp) / 8]; // Just ignore mips
-                Array.Copy(bitmap.RawData, tempData, tempData.Length);
+                if (bitmap.Bpp == 4 && bitmap.WiiAlphaNumber == 4)
+                {
+                    // Hidden alpha texture after mips... ugh
+                    var dxSize = (bitmap.Width * bitmap.Height * bitmap.Bpp) / 8;
+
+                    var rgbData = new byte[dxSize];
+                    var alphaData = new byte[dxSize];
+
+                    Array.Copy(bitmap.RawData, 0, rgbData, 0, rgbData.Length);
+                    Array.Copy(bitmap.RawData, bitmap.RawData.Length - rgbData.Length, alphaData, 0, alphaData.Length);
+
+                    // Remap blocks to DXT1
+                    Texture.TPL.TPLToDXT1(bitmap.Width, bitmap.Height, rgbData);
+                    Texture.TPL.TPLToDXT1(bitmap.Width, bitmap.Height, alphaData);
+
+                    // Decode both images
+                    var decodedImage = DecodeDxImage(rgbData, bitmap.Width, bitmap.Height, 0, DxEncoding.DXGI_FORMAT_BC1_UNORM);
+                    var decodedAlpha = DecodeDxImage(alphaData, bitmap.Width, bitmap.Height, 0, DxEncoding.DXGI_FORMAT_BC1_UNORM);
 
-                if (bitmap.Bpp == 4)
+                    // Combine alpha channel
+                    for (int i = 0; i < decodedImage.Length; i += 4)
+                    {
+                        // Load from green channel
+                        decodedImage[i + 3] = decodedAlpha[i + 1];
+                    }
+
+                    return decodedImage;
+                }
+                else if (bitmap.Bpp == 4)
                 {
+                    var tempData = new byte[((bitmap.Width * bitmap.Height) * bitmap.Bpp) / 8]; // Just ignore mips
+                    Array.Copy(bitmap.RawData, tempData, tempData.Length);
+
                     Texture.TPL.TPLToDXT1(bitmap.Width, bitmap.Height, tempData);
                     return DecodeDxImage(tempData, bitmap.Width, bitmap.Height, 0, DxEncoding.DXGI_FORMAT_BC1_UNORM);
                 }
                 else
                 {
+                    var tempData = new byte[((bitmap.Width * bitmap.Height) * bitmap.Bpp) / 8]; // Just ignore mips
+                    Array.Copy(bitmap.RawData, tempData, tempData.Length);
+
                     // 8bpp wii texture is actually two DXT1 textures
                     var rgbData = tempData.AsSpan(0, tempData.Length / 2);
                     var alphaData = tempData.AsSpan(tempData.Length / 2, tempData.Length / 2);
@@ -70,6 +101,7 @@ public static byte[] ToRGBA(this HMXBitmap bitmap, SystemInfo info)
                     // Combine alpha channel
                     for (int i = 0; i < decodedImage.Length; i += 4)
                     {
+                        // Load from green channel
                         decodedImage[i + 3] = decodedAlpha[i + 1];
                     }
 

Src/Core/Mackiloha.App/ImageWrapper.cs

@@ -146,26 +146,7 @@ public void WriteToFile(string filePath)
     public byte[] AsRGBA()
     {
         var data = new byte[_image.Width * _image.Height * 4];
-
-        _image.ProcessPixelRows(accessor =>
-        {
-            for (int y = 0; y < accessor.Height; y++)
-            {
-                var row = accessor.GetRowSpan(y);
-
-                for (int x = 0; x < row.Length; x++)
-                {
-                    var i = ((y * _image.Width) + x) * 4;
-                    var pixel = row[x];
-
-                    data[i    ] = pixel.R;
-                    data[i + 1] = pixel.G;
-                    data[i + 2] = pixel.B;
-                    data[i + 3] = pixel.A;
-                }
-            }
-        });
-
+        _image.CopyPixelDataTo(data);
         return data;
     }
 
@@ -177,11 +158,11 @@ public byte[] AsDXT1()
         encoder.OutputOptions.Format = BCnEncoder.Shared.CompressionFormat.Bc1;
         encoder.OutputOptions.FileFormat = BCnEncoder.Shared.OutputFileFormat.Dds;
 
-        using var ms = new MemoryStream();
+        var data = new byte[(_image.Width * _image.Height) >> 1];
+        using var ms = new MemoryStream(new byte[data.Length + 128]);
         encoder.EncodeToStream(_image, ms);
 
         // Copy to array (definitely not the most efficient...)
-        var data = new byte[(_image.Width * _image.Height) >> 1];
         ms.Seek(128, SeekOrigin.Begin);
         ms.Read(data, 0, data.Length);
 
@@ -196,11 +177,11 @@ public byte[] AsDXT5()
         encoder.OutputOptions.Format = BCnEncoder.Shared.CompressionFormat.Bc3;
         encoder.OutputOptions.FileFormat = BCnEncoder.Shared.OutputFileFormat.Dds;
 
-        using var ms = new MemoryStream();
+        var data = new byte[_image.Width * _image.Height];
+        using var ms = new MemoryStream(new byte[data.Length + 128]);
         encoder.EncodeToStream(_image, ms);
 
         // Copy to array (definitely not the most efficient...)
-        var data = new byte[_image.Width * _image.Height];
         ms.Seek(128, SeekOrigin.Begin);
         ms.Read(data, 0, data.Length);
 

Src/Core/Mackiloha/Ark/ArkFile.cs

@@ -446,6 +446,11 @@ private static (int part, long offset) GetArkOffsetForEntry(long entryOffset, lo
         return (currentIdx, (entryOffset - currentOffset));
     }
 
+    public void WriteHeader()
+    {
+        WriteHeader(_arkPaths.First());
+    }
+
     public void WriteHeader(string path)
     {
         using var ms = new MemoryStream();
@@ -819,136 +824,68 @@ public void CommitChanges(bool writeHeader)
             _offsetEntries.Clear();
         }
 
-        List<EntryOffset> GetGaps()
-        {
-            List<EntryOffset> offsetGaps = new List<EntryOffset>();
-            long previousOffset = 0;
-
-            foreach (var offsetEntry in remainingOffsetEntries)
-            {
-                if (offsetEntry.Offset - previousOffset == 0)
-                {
-                    // No gap, continues
-                    previousOffset = offsetEntry.Offset + offsetEntry.Size;
-                    continue;
-                }
-
-                // Adds gap to list
-                long gapOffset = previousOffset;
-                int gapSize = (int)(offsetEntry.Offset - previousOffset);
-                offsetGaps.Add(new EntryOffset(gapOffset, gapSize));
-
-                previousOffset = offsetEntry.Offset + offsetEntry.Size;
-            }
-
-            return offsetGaps;
-        }
-
-        void CopyToArchive(string arkFile, long arkOffset, string entryFile)
+        static void CopyToArchive(Stream fsArk, long arkOffset, string entryFile)
         {
-            // TODO: Extract out of this
-            using (FileStream fsArk = File.OpenWrite(arkFile))
-            {
-                fsArk.Seek(arkOffset, SeekOrigin.Begin);
+            fsArk.Seek(arkOffset, SeekOrigin.Begin);
 
-                using (FileStream fsEntry = File.OpenRead(entryFile))
-                {
-                    fsEntry.CopyTo(fsArk);
-                }
-            }
+            using var fsEntry = File.OpenRead(entryFile);
+            fsEntry.CopyTo(fsArk);
         }
 
-        // TODO: Compare previousOffset to ark file size
-        List<EntryOffset> gaps = GetGaps();
-        var pendingEntries = _pendingEntries.Select(x => new { Length = new FileInfo(x.LocalFilePath).Length, Entry = x }).OrderBy(x => x.Length);
+        var pendingEntries = _pendingEntries
+            .Select(x => new { Length = new FileInfo(x.LocalFilePath).Length, Entry = x })
+            .OrderBy(x => x.Length);
 
         // Gets lengths of ark files
         // TODO: Also check if part is encrypted
         var arkSizes = GetPartSizes()
             .Select(x => x.size)
             .ToArray();
 
-        foreach (var pending in pendingEntries)
-        {
-            // Looks at smallest gaps first, selects first fit
-            var bestFit = gaps.OrderBy(x => x.Size).FirstOrDefault(x => x.Size >= pending.Length);
+        var lastPartStartOffset = arkSizes
+            .Reverse()
+            .Skip(1)
+            .Sum();
 
-            if (arkSizes.Length > 1)
-                bestFit = null; // TODO: Update for multi-part arks
+        using var lastPartStream = File.OpenWrite(_arkPaths.Last());
 
-            if (bestFit == null)
-            {
-                // Adds to end of last archive file
-                var lastEntry = remainingOffsetEntries
-                    .OrderByDescending(x => x.Offset + x.Size) // Ensures 0-length files don't conflict w/ regular files at same offset
-                    .FirstOrDefault();
-
-                long offset = (lastEntry != null)
-                    ? lastEntry.Offset + lastEntry.Size
-                    : Version switch
-                    {
-                        ArkVersion.V2 => arkSizes.First(),
-                        _ => 0
-                    };
-
-                long partOffset = offset - arkSizes
-                    .Reverse()
-                    .Skip(1)
-                    .Sum();
-
-                // Copies entry to ark file
-                CopyToArchive(_arkPaths.Last(), partOffset, pending.Entry.LocalFilePath);
-
-                // Get inflate size if v2 or lower
-                var inflateSize = GetInflateSize(pending.Entry.LocalFilePath);
-
-                // Adds ark offset entry
-                remainingOffsetEntries.Add(new OffsetArkEntry(offset, pending.Entry.FileName, pending.Entry.Directory, (uint)pending.Length, inflateSize, arkSizes.Length, partOffset));
-            }
-            else
-            {
-                (int partIdx, long partOffset) = GetArkOffsetForEntry(bestFit.Offset, arkSizes);
+        foreach (var pending in pendingEntries)
+        {
+            // Adds to end of last archive file
+            var lastEntry = remainingOffsetEntries
+                .OrderByDescending(x => x.Offset + x.Size) // Ensures 0-length files don't conflict w/ regular files at same offset
+                .FirstOrDefault();
+
+            long offset = (lastEntry != null)
+                ? lastEntry.Offset + lastEntry.Size
+                : Version switch
+                {
+                    ArkVersion.V2 => arkSizes.First(),
+                    _ => 0
+                };
 
-                // Copies entry to ark file (TODO: Calculate arkPath beforehand)
-                CopyToArchive(_arkPaths[partIdx + 1], partOffset, pending.Entry.LocalFilePath);
+            long partOffset = offset - lastPartStartOffset;
 
-                // Get inflate size if v2 or lower
-                var inflateSize = GetInflateSize(pending.Entry.LocalFilePath);
+            // Copies entry to ark file
+            CopyToArchive(lastPartStream, partOffset, pending.Entry.LocalFilePath);
 
-                // Adds ark offset entry
-                remainingOffsetEntries.Add(new OffsetArkEntry(bestFit.Offset, pending.Entry.FileName, pending.Entry.Directory, (uint)pending.Length, inflateSize, partIdx + 1, partOffset));
+            // Get inflate size if v2 or lower
+            var inflateSize = GetInflateSize(pending.Entry.LocalFilePath);
 
-                // Updates gap entry
-                if (bestFit.Size == pending.Length)
-                {
-                    // Remove gap
-                    gaps.Remove(bestFit);
-                }
-                else
-                {
-                    // Updates values
-                    bestFit.Offset += pending.Length;
-                    bestFit.Size -= (int)pending.Length;
-                }
-            }
+            // Adds ark offset entry
+            remainingOffsetEntries.Add(new OffsetArkEntry(offset, pending.Entry.FileName, pending.Entry.Directory, (uint)pending.Length, inflateSize, arkSizes.Length, partOffset));
         }
 
         // Updates archive entries
         _pendingEntries.Clear();
         _offsetEntries.Clear();
         _offsetEntries.AddRange(remainingOffsetEntries);
 
-        // Re-writes header file
-        if (writeHeader && (int)Version >= 3)
-            WriteHeader(_arkPaths[0]);
-        else if ((int)Version < 3)
+        // Writes header file
+        if (writeHeader || (int)Version < 3)
         {
-            // Update header
-            using var fs = File.OpenWrite(_arkPaths.First());
-            WriteHeader(fs);
+            WriteHeader(_arkPaths.First());
         }
-
-        // TODO: Add an output log
     }
 
     protected uint GetInflateSize(string path)
@@ -958,11 +895,11 @@ protected uint GetInflateSize(string path)
             // Don't care when version is 3 or above
             return 0;
         }
-        else if (Regex.IsMatch(path, "(?i).gz$"))
+        else if (path.EndsWith(".gz", StringComparison.InvariantCultureIgnoreCase))
         {
             return GetGZipInflateSize(path);
         }
-        else if (path.EndsWith(".z") || path.EndsWith(".Z"))
+        else if (path.EndsWith(".z", StringComparison.InvariantCultureIgnoreCase))
         {
             return GetZlibInflateSize(path);
         }

Src/Core/Mackiloha/HMXBitmap.cs

@@ -9,6 +9,7 @@ public class HMXBitmap : ISerializable
     public int Width { get; set; }
     public int Height { get; set; }
     public int BPL { get; set; }
+    public int WiiAlphaNumber { get; set; }
 
     public byte[] RawData { get; set; }
 }

Src/Core/Mackiloha/IO/Serializers/HMXBitmapSerializer.cs

@@ -20,9 +20,10 @@ public override void ReadFromStream(AwesomeReader ar, ISerializable data)
         bitmap.Width = ar.ReadUInt16();
         bitmap.Height = ar.ReadUInt16();
         bitmap.BPL = ar.ReadUInt16();
+        bitmap.WiiAlphaNumber = ar.ReadUInt16(); // Assuming u16
 
-        ar.BaseStream.Position += 19; // Skips zeros
-        bitmap.RawData = ar.ReadBytes(CalculateTextureByteSize(bitmap.Encoding, bitmap.Width, bitmap.Height, bitmap.Bpp, bitmap.MipMaps));
+        ar.BaseStream.Position += 17; // Skips zeros
+        bitmap.RawData = ar.ReadBytes(CalculateTextureByteSize(bitmap.Encoding, bitmap.Width, bitmap.Height, bitmap.Bpp, bitmap.MipMaps, bitmap.WiiAlphaNumber));
     }
 
     public override void WriteToStream(AwesomeWriter aw, ISerializable data)
@@ -46,7 +47,7 @@ public override void WriteToStream(AwesomeWriter aw, ISerializable data)
         aw.Write(bytes);
     }
 
-    private int CalculateTextureByteSize(int encoding, int w, int h, int bpp, int mips)
+    private int CalculateTextureByteSize(int encoding, int w, int h, int bpp, int mips, int wiiAlphaNum = 0)
     {
         int bytes = 0;
 
@@ -62,6 +63,12 @@ private int CalculateTextureByteSize(int encoding, int w, int h, int bpp, int mi
                 // Each color is 32 bits 
                 bytes += (bpp == 4 || bpp == 8) ? 1 << (bpp + 2) : 0;
                 break;
+            case 72: // Wii 4bpp texture
+                if (wiiAlphaNum == 4)
+                {
+                    bytes += (w * h * bpp) / 8;
+                }
+                break;
         }
 
         while (mips >= 0)

version.txt

@@ -1 +1 @@
-1.3.0
+1.3.1