From eb7d127a6d78bb1d891fe6a6cdc06cd2b2573615 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 13 Oct 2017 00:13:44 -0700 Subject: [PATCH 1/3] Add tests for packing basic datatypes --- test/runtests.jl | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index f28482f..45ed1f4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -108,13 +108,28 @@ end end @testset "pack()" begin + # Pack simple types + buf = IOBuffer() + pack(buf, UInt8(1)) + pack(buf, Int16(2)) + pack(buf, UInt32(4)) + pack(buf, Int64(8)) + pack(buf, UInt128(16)) + @test position(buf) == 1 + 2 + 4 + 8 + 16 + seekstart(buf) + @test read(buf, UInt8) === UInt8(1) + @test read(buf, Int16) === Int16(2) + @test read(buf, UInt32) === UInt32(4) + @test read(buf, Int64) === Int64(8) + @test read(buf, UInt128) === UInt128(16) + # Pack a simple object buf = IOBuffer() tu = TwoUInts(2, 3) pack(buf, tu) # Test that the stream looks reasonable - @test position(buf) == Core.sizeof(TwoUInts) + @test position(buf) == sizeof(TwoUInts) seekstart(buf) @test read(buf, UInt) == 2 @test read(buf, UInt) == 3 From 5487d39f3e63a4d70f910ff9fef02a9f0ef80aab Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 13 Oct 2017 00:14:55 -0700 Subject: [PATCH 2/3] Fix depwarn --- src/StructIO.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/StructIO.jl b/src/StructIO.jl index a0b61fa..2e8c790 100644 --- a/src/StructIO.jl +++ b/src/StructIO.jl @@ -198,7 +198,7 @@ last argument is a packing strategy, used to determine the layout of the data in memory. All `Packed` objects recurse until bitstypes objects are eventually reached, at which point `Default` packing is identical to `Packed` behavior. """ -function unsafe_pack{T}(io, source::Ref{T}, endianness, ::Type{Default}) +function unsafe_pack(io, source::Ref{T}, endianness, ::Type{Default}) where {T} sz = packed_sizeof(T) if !needs_bswap(endianness) # If we don't need to bswap, just write directly from `source` @@ -253,7 +253,7 @@ function unsafe_unpack(io, T, target, endianness, ::Type{Packed}) end # `Packed` packing strategy override for `unsafe_pack` -function unsafe_pack{T}(io, source::Ref{T}, endianness, ::Type{Packed}) +function unsafe_pack(io, source::Ref{T}, endianness, ::Type{Packed}) where {T} # If this type cannot be subdivided, packing strategy means nothing, so # hand it off to the `Default` packing strategy method if @compat fieldcount(T) == 0 @@ -297,7 +297,7 @@ no byteswapping will occur. If `endianness` is `:LittleEndian` or `:BigEndian`, byteswapping will occur if the endianness of the host system does not match the endianness of `io`. """ -function pack{T}(io::IO, source::T, endianness::Symbol = :NativeEndian) +function pack(io::IO, source::T, endianness::Symbol = :NativeEndian) where {T} r = Ref{T}(source) packstrat = @compat fieldcount(T) == 0 ? Default : packing_strategy(T) unsafe_pack(io, r, endianness, packstrat) From e6626d1ac87033430e17aaeafb97dfd873ee3e49 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Fri, 13 Oct 2017 00:50:01 -0700 Subject: [PATCH 3/3] Rework `unsafe_unpack()` to actually work for byteswapped bitstypes --- src/StructIO.jl | 29 +++++++++++++++-------------- test/runtests.jl | 28 +++++++++++++++------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/src/StructIO.jl b/src/StructIO.jl index 2e8c790..0a85709 100644 --- a/src/StructIO.jl +++ b/src/StructIO.jl @@ -147,7 +147,7 @@ Unpack an object of type `T` from `io` into `target`, byte-swapping if packed structs recurse until bitstypes objects are eventually reached, at which point `Default` packing is the only behavior. """ -function unsafe_unpack(io, T, target, endianness, ::Type{Default}) +function unsafe_unpack(io, ::Type{T}, target, endianness, ::Type{Default}) where {T} sz = Core.sizeof(T) if !needs_bswap(endianness) @@ -160,18 +160,19 @@ function unsafe_unpack(io, T, target, endianness, ::Type{Default}) # Special case small sizes, LLVM should turn this into a jump table if sz == 1 elseif sz == 2 - ptr = Base.unsafe_convert(Ptr{UInt16}, target) + ptr = Base.unsafe_convert(Ptr{T}, target) unsafe_store!(ptr, bswap(unsafe_load(ptr))) elseif sz == 4 - ptr = Base.unsafe_convert(Ptr{UInt32}, target) + ptr = Base.unsafe_convert(Ptr{T}, target) unsafe_store!(ptr, bswap(unsafe_load(ptr))) elseif sz == 8 - ptr = Base.unsafe_convert(Ptr{UInt64}, target) + ptr = Base.unsafe_convert(Ptr{T}, target) unsafe_store!(ptr, bswap(unsafe_load(ptr))) else # Otherwise, for large primitive objects, fall back to our # `bswap!()` method which will swap in-place - bswap!(Base.unsafe_convert(Ptr{UInt8}, target), sz) + void_ptr = Base.unsafe_convert(Ptr{Void}, target) + bswap!(Base.unsafe_convert(Ptr{UInt8}, void_ptr), sz) end else # If we need to bswap, but it's not a primitive type, recurse! @@ -206,24 +207,24 @@ function unsafe_pack(io, source::Ref{T}, endianness, ::Type{Default}) where {T} elseif @compat fieldcount(T) == 0 # Hopefully, LLVM turns this into a jump list for us if sz == 1 - unsafe_write(io, source[]) + write(io, source[]) elseif sz == 2 - ptr = Base.unsafe_convert(Ptr{UInt16}, source) - unsafe_write(io, bswap(unsafe_load(ptr)), sz) + ptr = Base.unsafe_convert(Ptr{T}, source) + write(io, bswap(unsafe_load(ptr))) elseif sz == 4 - ptr = Base.unsafe_convert(Ptr{UInt32}, source) - unsafe_write(io, bswap(unsafe_load(ptr)), sz) + ptr = Base.unsafe_convert(Ptr{T}, source) + write(io, bswap(unsafe_load(ptr))) elseif sz == 8 - ptr = Base.unsafe_convert(Ptr{UInt64}, source) - unsafe_write(io, bswap(unsafe_load(ptr)), sz) + ptr = Base.unsafe_convert(Ptr{T}, source) + write(io, bswap(unsafe_load(ptr))) else # If we must bswap something of unknown size, copy first so as # to not clobber `source`, then bswap, then write - ptr = Base.unsafe_convert(Ptr{UInt8}, copy(source)) + void_ptr = Base.unsafe_convert(Ptr{Void}, Ref{T}(copy(source[]))) + ptr = Base.unsafe_convert(Ptr{UInt8}, void_ptr) bswap!(ptr, sz) unsafe_write(io, ptr, sz) end - @show position(io), T, sz else # If we need to bswap, but it's not a primitive type, recurse! for i = 1:@compat fieldcount(T) diff --git a/test/runtests.jl b/test/runtests.jl index 45ed1f4..11497e3 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -109,19 +109,21 @@ end @testset "pack()" begin # Pack simple types - buf = IOBuffer() - pack(buf, UInt8(1)) - pack(buf, Int16(2)) - pack(buf, UInt32(4)) - pack(buf, Int64(8)) - pack(buf, UInt128(16)) - @test position(buf) == 1 + 2 + 4 + 8 + 16 - seekstart(buf) - @test read(buf, UInt8) === UInt8(1) - @test read(buf, Int16) === Int16(2) - @test read(buf, UInt32) === UInt32(4) - @test read(buf, Int64) === Int64(8) - @test read(buf, UInt128) === UInt128(16) + for endian in [:BigEndian, :LittleEndian] + buf = IOBuffer() + pack(buf, UInt8(1), endian) + pack(buf, Int16(2), endian) + pack(buf, UInt32(4), endian) + pack(buf, Int64(8), endian) + pack(buf, UInt128(16), endian) + @test position(buf) == 1 + 2 + 4 + 8 + 16 + seekstart(buf) + @test unpack(buf, UInt8, endian) === UInt8(1) + @test unpack(buf, Int16, endian) === Int16(2) + @test unpack(buf, UInt32, endian) === UInt32(4) + @test unpack(buf, Int64, endian) === Int64(8) + @test unpack(buf, UInt128, endian) === UInt128(16) + end # Pack a simple object buf = IOBuffer()