diff --git a/README.md b/README.md index 2acf9c32..2770681f 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,7 @@ Everything will immediately show up in ImHex's Content Store and gets bundled wi | Lua 5.3 | | [`patterns/lua53.hexpat`](patterns/lua53.hexpat) | Lua 5.3 bytecode | | Lua 5.4 | | [`patterns/lua54.hexpat`](patterns/lua54.hexpat) | Lua 5.4 bytecode | | LCE Savefile | | [`patterns/lcesave.hexpat`](patterns/lcesave.hexpat) | Minecraft Legacy Console Edition save file | +| LZNT1 | | [`patterns/lznt1.hexpat`](patterns/lznt1.hexpat) | LZNT1 compressed data format | | Mach-O | `application/x-mach-binary` | [`patterns/macho.hexpat`](patterns/macho.hexpat) | Mach-O executable | | MIDI | `audio/midi` | [`patterns/midi.hexpat`](patterns/midi.hexpat) | MIDI header, event fields provided | | MiniDump | `application/x-dmp` | [`patterns/minidump.hexpat`](patterns/minidump.hexpat) | Windows MiniDump files | diff --git a/patterns/lznt1.hexpat b/patterns/lznt1.hexpat new file mode 100644 index 00000000..dfbfe441 --- /dev/null +++ b/patterns/lznt1.hexpat @@ -0,0 +1,206 @@ +#pragma description LZNT1 +#pragma endian little +#pragma pattern_limit 1000000 + +/* + * References: + * https://github.com/libyal/libfwnt/blob/main/documentation/Compression%20methods.asciidoc + * https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-xca/5655f4a3-6ba4-489b-959f-e1f407c52f15 + * https://github.com/tuxera/ntfs-3g/blob/edge/libntfs-3g/compress.c + */ + +import std.core; +import std.io; +import std.mem; +import std.sys; + +using BitfieldOrder = std::core::BitfieldOrder; +bool createDecompressedSection in; +std::mem::Section decompressedSection = 0; +u128 uncompressedDataSize = 0; +u128 maxUnitSize = 4096; + +fn appendData(ref auto data) { + if (createDecompressedSection) { + std::mem::copy_value_to_section(data, + decompressedSection, + std::mem::get_section_size(decompressedSection)); + } + + uncompressedDataSize += sizeof(data); +}; + +struct Value { + u8 value; +}; + +fn appendU8(u8 data) { + Value value; + value.value = data; + appendData(value); +}; + +bitfield Flag { + bool A : 1; + bool B : 1; + bool C : 1; + bool D : 1; + bool E : 1; + bool F : 1; + bool G : 1; + bool H : 1; +} [[bitfield_order(BitfieldOrder::LeastToMostSignificant, 8)]]; + +bitfield CompressedTuple { + std::assert(lengthSize >= 4 && lengthSize <= 12, + std::format("lengthSize has an invalid value {}. Must be between 4 and 12.", lengthSize)); + std::assert(displacementSize >= 4 && displacementSize <= 12, + std::format("displacementSizehas an invalid value {}. Must be between 4 and 12.", displacementSize)); + std::assert((lengthSize + displacementSize) == 16, + std::format("lengthSize {} and displacementSize {} must add up to 16.", lengthSize, displacementSize)); + unsigned length : lengthSize; + unsigned displacement : displacementSize; + u16 actualLength = u16(length) + 3 [[export]]; + s16 actualDisplacement = -1 * (s16(displacement) + 1) [[export]]; +} [[bitfield_order(BitfieldOrder::LeastToMostSignificant, 16)]]; + +fn calculateLengthSize() { + s8 lengthSize = 12; + + for (u128 i = uncompressedDataSize - 1, i >= 0x10, i = i >> 1) { + lengthSize = lengthSize - 1; + } + + if (lengthSize < 4) { + lengthSize = 4; + } + + return u8(lengthSize); +}; + +fn copySequentially(std::mem::Section section, u128 sourcePos, u128 destinationPos, u128 length) { + for (u128 i = 0, i < length, i = i + 1) { + std::mem::copy_section_to_section(section, + sourcePos + i, + section, + destinationPos + i, + 1); + } +}; + +struct Tuple { + std::assert(uncompressedDataSize > 0, + "uncompressedDataSize must be greater than zero" + + " because otherwise there would be no data to backrefrence!"); + u8 ls = calculateLengthSize(); + CompressedTuple ct[[inline]]; + std::assert((-1 * ct.actualDisplacement) <= uncompressedDataSize, + std::format("The actualDisplacement {} is referencing data before the beginning" + + " of the current decompressed chunk! Current decompressed size is {}.", + ct.actualDisplacement, + uncompressedDataSize)); + + if (createDecompressedSection) { + u128 destinationPos = std::mem::get_section_size(decompressedSection); + u128 destinationBackrefPos = destinationPos + ct.actualDisplacement; + u128 maxNonOverlap = destinationPos - destinationBackrefPos; + + if (ct.actualLength <= maxNonOverlap) { // Not overlapping + std::mem::copy_section_to_section(decompressedSection, + destinationBackrefPos, + decompressedSection, + destinationPos, + ct.actualLength); + } else { // Overlapping + // Copy non-overlapping part + std::mem::copy_section_to_section(decompressedSection, + destinationBackrefPos, + decompressedSection, + destinationPos, + maxNonOverlap); + // Copy overlapping part + destinationPos += maxNonOverlap; + destinationBackrefPos += maxNonOverlap; + copySequentially(decompressedSection, destinationBackrefPos, + destinationPos, ct.actualLength - maxNonOverlap); + } + } + + uncompressedDataSize += ct.actualLength; + std::assert(uncompressedDataSize <= maxUnitSize, + std::format("uncompressedDataSize {} is larger than the maximum allowed size of {}.", + uncompressedDataSize, + maxUnitSize)); +}; + +struct FlagGroup { + Flag flag [[comment("Each bit represents whether a data element in a group of 8 is compressed " + + "with the first value being the least significant bit.")]]; + // (up to) 8 data elements + if ($ >= endOffset) break; + if (flag.A) { Tuple A; } else { u8 A; appendU8(A); } + if ($ >= endOffset) break; + if (flag.B) { Tuple B; } else { u8 B; appendU8(B); } + if ($ >= endOffset) break; + if (flag.C) { Tuple C; } else { u8 C; appendU8(C); } + if ($ >= endOffset) break; + if (flag.D) { Tuple D; } else { u8 D; appendU8(D); } + if ($ >= endOffset) break; + if (flag.E) { Tuple E; } else { u8 E; appendU8(E); } + if ($ >= endOffset) break; + if (flag.F) { Tuple F; } else { u8 F; appendU8(F); } + if ($ >= endOffset) break; + if (flag.G) { Tuple G; } else { u8 G; appendU8(G); } + if ($ >= endOffset) break; + if (flag.H) { Tuple H; } else { u8 H; appendU8(H); } +}; + +bitfield ChunkHeader { + unsigned chunkDataSize : 12 [[comment("The actual value stored is the chunk data size - 1.")]]; + unsigned signatureValue : 3 [[comment("The value is always 3 except in an all-zero chunk header.")]]; + bool isCompressed : 1; +} [[bitfield_order(BitfieldOrder::LeastToMostSignificant, 16)]]; + +struct Chunk { + auto headerPos = $; + ChunkHeader header; + + /* An all-zero chunk header indicates the end of an LZNT1 compression stream. + Might not be present if there is not enough space to store it. */ + if (header.chunkDataSize == 0 + && header.signatureValue == 0 + && !header.isCompressed) { + break; + } + + std::assert_warn(header.signatureValue == 3, + std::format("ChunkHeader @ {:#x} has a signatureValue other than 3: {}", + headerPos, header.signatureValue)); + + u128 actualChunkDataSize = u128(header.chunkDataSize) + 1 [[export]]; + + if (header.isCompressed) { + auto currentEndOffset = $ + actualChunkDataSize; + uncompressedDataSize = 0; + FlagGroup data[while($ < currentEndOffset)]; + std::assert($ == currentEndOffset, + std::format("Invalid size of Chunk @ {:#x}.", headerPos)); + } else { + std::assert_warn(actualChunkDataSize == maxUnitSize, + std::format("actualChunkDataSize {} must be equal to maxUnitSize {}.", + actualChunkDataSize, + maxUnitSize)); + u8 data[actualChunkDataSize]; + appendData(data); + } +}; + +struct LZNT1 { + if (createDecompressedSection) { + decompressedSection = std::mem::create_section(std::format("decompressed @ {:#x}", $)); + } + + Chunk chunks[while($ < sizeof($))]; +}; + +LZNT1 lznt1 @ 0x00; \ No newline at end of file diff --git a/tests/patterns/test_data/lznt1.hexpat.lznt1 b/tests/patterns/test_data/lznt1.hexpat.lznt1 new file mode 100644 index 00000000..07d4fa2c Binary files /dev/null and b/tests/patterns/test_data/lznt1.hexpat.lznt1 differ