From 11cff425f9e0682f6c52def63a988bfc6a3d3f68 Mon Sep 17 00:00:00 2001 From: nhz2 Date: Fri, 6 Sep 2024 22:20:44 -0400 Subject: [PATCH 1/8] Add pledgeinsize --- src/compression.jl | 28 +++++++++++++++++++++++----- src/libzstd.jl | 16 ++-------------- 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/compression.jl b/src/compression.jl index cabc3f9..693c41d 100644 --- a/src/compression.jl +++ b/src/compression.jl @@ -101,13 +101,31 @@ function TranscodingStreams.finalize(codec::ZstdCompressor) return end -function TranscodingStreams.startproc(codec::ZstdCompressor, mode::Symbol, error::Error) - code = reset!(codec.cstream, 0 #=unknown source size=#) +function TranscodingStreams.startproc(codec::ZstdCompressor, mode::Symbol, error::Error)::Symbol + code = reset!(codec.cstream) if iserror(code) - error[] = ErrorException("zstd error") - return :error + error[] = ErrorException("zstd error resetting compression context") + :error + else + :ok + end +end + +if isdefined(TranscodingStreams, :pledgeinsize) + function TranscodingStreams.pledgeinsize(codec::ZstdCompressor, insize::Int64, error::Error)::Symbol + srcsize = if signbit(insize) + ZSTD_CONTENTSIZE_UNKNOWN + else + Culonglong(insize) + end + code = LibZstd.ZSTD_CCtx_setPledgedSrcSize(codec.cstream, srcsize) + if iserror(code) + error[] = ErrorException("zstd error setting pledged source size") + :error + else + :ok + end end - return :ok end function TranscodingStreams.process(codec::ZstdCompressor, input::Memory, output::Memory, error::Error) diff --git a/src/libzstd.jl b/src/libzstd.jl index 79d021d..0112b4e 100644 --- a/src/libzstd.jl +++ b/src/libzstd.jl @@ -60,23 +60,11 @@ function initialize!(cstream::CStream, level::Integer) return LibZstd.ZSTD_initCStream(cstream, level) end -function reset!(cstream::CStream, srcsize::Integer) - # ZSTD_resetCStream is deprecated - # https://github.com/facebook/zstd/blob/9d2a45a705e22ad4817b41442949cd0f78597154/lib/zstd.h#L2253-L2272 +function reset!(cstream::CStream) res = LibZstd.ZSTD_CCtx_reset(cstream, LibZstd.ZSTD_reset_session_only) - if iserror(res) - return res - end - if srcsize == 0 - # From zstd.h: - # Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but - # ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be - # explicitly specified. - srcsize = ZSTD_CONTENTSIZE_UNKNOWN - end reset!(cstream.ibuffer) reset!(cstream.obuffer) - return LibZstd.ZSTD_CCtx_setPledgedSrcSize(cstream, srcsize) + return res end """ From e0c0238460ddacfdefe696f36f9c6cfa8a52b47d Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 5 Oct 2024 12:04:54 -0400 Subject: [PATCH 2/8] Add tests for pledgeinsize --- .github/workflows/Downstream.yml | 1 + .github/workflows/Upstream.yml | 31 +++++++++++++++++++++++++++++++ test/runtests.jl | 27 +++++++++++++++++++++++++-- 3 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/Upstream.yml diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml index 880220c..adbf164 100644 --- a/.github/workflows/Downstream.yml +++ b/.github/workflows/Downstream.yml @@ -31,6 +31,7 @@ jobs: Pkg.Registry.update() Pkg.activate(;temp=true) # force it to use this PR's version of the package + ENV["JULIA_PKG_DEVDIR"]= mktempdir() Pkg.develop([ PackageSpec(path="."), PackageSpec(name="${{ matrix.package }}"), diff --git a/.github/workflows/Upstream.yml b/.github/workflows/Upstream.yml new file mode 100644 index 0000000..cd60bce --- /dev/null +++ b/.github/workflows/Upstream.yml @@ -0,0 +1,31 @@ +name: Upstream +on: + push: + branches: [master] + tags: [v*] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@v2 + with: + version: 1 + arch: x64 + - uses: julia-actions/julia-buildpkg@latest + - name: Load the upstream packages and run the tests + shell: julia --color=yes {0} + run: | + using Pkg + Pkg.Registry.update() + Pkg.activate(;temp=true) + # force it to use this PR's version of the package + ENV["JULIA_PKG_DEVDIR"]= mktempdir() + Pkg.develop([ + PackageSpec(path="."), + PackageSpec(name="TranscodingStreams"), + ]) + Pkg.update() + Pkg.test("CodecZstd") diff --git a/test/runtests.jl b/test/runtests.jl index 73cf82f..c2c195d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -130,9 +130,17 @@ include("utils.jl") @test CodecZstd.find_decompressed_size(v) == 22 codec = ZstdCompressor - buffer3 = transcode(codec, b"Hello") - buffer4 = transcode(codec, b"World!") + sink = IOBuffer() + s = TranscodingStream(codec(), sink; stop_on_end=true) + write(s, b"Hello") + close(s) + buffer3 = take!(sink) @test CodecZstd.find_decompressed_size(buffer3) == CodecZstd.ZSTD_CONTENTSIZE_UNKNOWN + sink = IOBuffer() + s = TranscodingStream(codec(), sink; stop_on_end=true) + write(s, b"Hello") + close(s) + buffer4 = take!(sink) @test CodecZstd.find_decompressed_size(buffer4) == CodecZstd.ZSTD_CONTENTSIZE_UNKNOWN write(iob, buffer1) @@ -156,6 +164,21 @@ include("utils.jl") @test CodecZstd.find_decompressed_size(v) == CodecZstd.ZSTD_CONTENTSIZE_ERROR end + @testset "pledgeinsize" begin + if isdefined(TranscodingStreams, :pledgeinsize) + # when pledgeinsize is available transcode should save the + # decompressed size in a header + for n in [0:30; 1000; 1000000; 10000000;] + a = transcode(ZstdCompressor, rand(UInt8, n)) + @test CodecZstd.find_decompressed_size(a) == n + a = transcode(ZstdCompressor, rand(0x00:0x01, n)) + @test CodecZstd.find_decompressed_size(a) == n + a = transcode(ZstdCompressor, zeros(UInt8, n)) + @test CodecZstd.find_decompressed_size(a) == n + end + end + end + include("compress_endOp.jl") include("static_only_tests.jl") From b2c4e97cba099bb421575a0ef4b3c0175b00b99f Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 5 Oct 2024 12:49:50 -0400 Subject: [PATCH 3/8] test errors --- src/compression.jl | 5 ++++- test/runtests.jl | 54 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/compression.jl b/src/compression.jl index 11e014b..6a2b7b8 100644 --- a/src/compression.jl +++ b/src/compression.jl @@ -112,6 +112,9 @@ end if isdefined(TranscodingStreams, :pledgeinsize) function TranscodingStreams.pledgeinsize(codec::ZstdCompressor, insize::Int64, error::Error)::Symbol + if codec.cstream.ptr == C_NULL + Base.error("`startproc` must be called before `pledgeinsize`") + end srcsize = if signbit(insize) ZSTD_CONTENTSIZE_UNKNOWN else @@ -129,7 +132,7 @@ end function TranscodingStreams.process(codec::ZstdCompressor, input::Memory, output::Memory, error::Error) if codec.cstream.ptr == C_NULL - error("startproc must be called before process") + Base.error("`startproc` must be called before `process`") end cstream = codec.cstream ibuffer_starting_pos = UInt(0) diff --git a/test/runtests.jl b/test/runtests.jl index c2c195d..b705864 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -168,13 +168,47 @@ include("utils.jl") if isdefined(TranscodingStreams, :pledgeinsize) # when pledgeinsize is available transcode should save the # decompressed size in a header - for n in [0:30; 1000; 1000000; 10000000;] - a = transcode(ZstdCompressor, rand(UInt8, n)) - @test CodecZstd.find_decompressed_size(a) == n - a = transcode(ZstdCompressor, rand(0x00:0x01, n)) - @test CodecZstd.find_decompressed_size(a) == n - a = transcode(ZstdCompressor, zeros(UInt8, n)) - @test CodecZstd.find_decompressed_size(a) == n + for n in [0:30; 1000; 1000000;] + v = transcode(ZstdCompressor, rand(UInt8, n)) + @test CodecZstd.find_decompressed_size(v) == n + end + + # Test what happens if pledgeinsize promise is broken + d1 = zeros(UInt8, 10000) + d2 = zeros(UInt8, 10000) + GC.@preserve d1 d2 begin + @testset "too many bytes" begin + m1 = TranscodingStreams.Memory(pointer(d1), 1000) + m2 = TranscodingStreams.Memory(pointer(d2), 1000) + codec = ZstdCompressor() + e = TranscodingStreams.Error() + @test TranscodingStreams.startproc(codec, :read, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, 10, e) === :ok + @test TranscodingStreams.process(codec, m1, m2, e) === (0, 0, :error) + @test e[] == ErrorException("zstd error: Src size is incorrect") + end + @testset "too few bytes" begin + m1 = TranscodingStreams.Memory(pointer(d1), 10) + m2 = TranscodingStreams.Memory(pointer(d2), 1000) + codec = ZstdCompressor() + e = TranscodingStreams.Error() + @test TranscodingStreams.startproc(codec, :read, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, 10000, e) === :ok + @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok + m1 = TranscodingStreams.Memory(pointer(d1), 0) + @test TranscodingStreams.process(codec, m1, m2, e)[3] === :error + @test e[] == ErrorException("zstd error: Src size is incorrect") + end + @testset "set pledgeinsize after process" begin + m1 = TranscodingStreams.Memory(pointer(d1), 1000) + m2 = TranscodingStreams.Memory(pointer(d2), 1000) + codec = ZstdCompressor() + e = TranscodingStreams.Error() + @test TranscodingStreams.startproc(codec, :read, e) === :ok + @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok + @test TranscodingStreams.pledgeinsize(codec, 10000, e) === :error + @test e[] == ErrorException("zstd error setting pledged source size") + end end end end @@ -218,6 +252,12 @@ include("utils.jl") TranscodingStreams.finalize(codec) data = [0x00,0x01] GC.@preserve data let m = TranscodingStreams.Memory(pointer(data), length(data)) + if isdefined(TranscodingStreams, :pledgeinsize) + try + TranscodingStreams.pledgeinsize(codec, 10, TranscodingStreams.Error()) + catch + end + end try TranscodingStreams.expectedsize(codec, m) catch From 783ecc9e20bf6dd7c0bbd8c764169511de97f260 Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 5 Oct 2024 13:00:21 -0400 Subject: [PATCH 4/8] Add coverage for upstream tests --- .github/workflows/Upstream.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/Upstream.yml b/.github/workflows/Upstream.yml index cd60bce..137ec54 100644 --- a/.github/workflows/Upstream.yml +++ b/.github/workflows/Upstream.yml @@ -28,4 +28,9 @@ jobs: PackageSpec(name="TranscodingStreams"), ]) Pkg.update() - Pkg.test("CodecZstd") + Pkg.test("CodecZstd"; coverage=true) + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info From e1f070b1874c8ca4cf1bb51c7b2444c94ce86d60 Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 5 Oct 2024 14:15:08 -0400 Subject: [PATCH 5/8] test unknown pledgeinsize --- test/runtests.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/runtests.jl b/test/runtests.jl index b705864..f17ea44 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -186,6 +186,7 @@ include("utils.jl") @test TranscodingStreams.pledgeinsize(codec, 10, e) === :ok @test TranscodingStreams.process(codec, m1, m2, e) === (0, 0, :error) @test e[] == ErrorException("zstd error: Src size is incorrect") + TranscodingStreams.finalize(codec) end @testset "too few bytes" begin m1 = TranscodingStreams.Memory(pointer(d1), 10) @@ -198,6 +199,7 @@ include("utils.jl") m1 = TranscodingStreams.Memory(pointer(d1), 0) @test TranscodingStreams.process(codec, m1, m2, e)[3] === :error @test e[] == ErrorException("zstd error: Src size is incorrect") + TranscodingStreams.finalize(codec) end @testset "set pledgeinsize after process" begin m1 = TranscodingStreams.Memory(pointer(d1), 1000) @@ -208,6 +210,17 @@ include("utils.jl") @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok @test TranscodingStreams.pledgeinsize(codec, 10000, e) === :error @test e[] == ErrorException("zstd error setting pledged source size") + TranscodingStreams.finalize(codec) + end + @testset "set unknown pledgeinsize" begin + m1 = TranscodingStreams.Memory(pointer(d1), 1000) + m2 = TranscodingStreams.Memory(pointer(d2), 1000) + codec = ZstdCompressor() + e = TranscodingStreams.Error() + @test TranscodingStreams.startproc(codec, :read, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, -1, e) === :ok + @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok + TranscodingStreams.finalize(codec) end end end From 2a5b290a4731684c857402d15fc668529790b716 Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 5 Oct 2024 15:12:56 -0400 Subject: [PATCH 6/8] fix tests for 32 bit --- test/runtests.jl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index f17ea44..78402cc 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -183,7 +183,7 @@ include("utils.jl") codec = ZstdCompressor() e = TranscodingStreams.Error() @test TranscodingStreams.startproc(codec, :read, e) === :ok - @test TranscodingStreams.pledgeinsize(codec, 10, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, Int64(10), e) === :ok @test TranscodingStreams.process(codec, m1, m2, e) === (0, 0, :error) @test e[] == ErrorException("zstd error: Src size is incorrect") TranscodingStreams.finalize(codec) @@ -194,7 +194,7 @@ include("utils.jl") codec = ZstdCompressor() e = TranscodingStreams.Error() @test TranscodingStreams.startproc(codec, :read, e) === :ok - @test TranscodingStreams.pledgeinsize(codec, 10000, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, Int64(10000), e) === :ok @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok m1 = TranscodingStreams.Memory(pointer(d1), 0) @test TranscodingStreams.process(codec, m1, m2, e)[3] === :error @@ -208,7 +208,7 @@ include("utils.jl") e = TranscodingStreams.Error() @test TranscodingStreams.startproc(codec, :read, e) === :ok @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok - @test TranscodingStreams.pledgeinsize(codec, 10000, e) === :error + @test TranscodingStreams.pledgeinsize(codec, Int64(10000), e) === :error @test e[] == ErrorException("zstd error setting pledged source size") TranscodingStreams.finalize(codec) end @@ -218,7 +218,7 @@ include("utils.jl") codec = ZstdCompressor() e = TranscodingStreams.Error() @test TranscodingStreams.startproc(codec, :read, e) === :ok - @test TranscodingStreams.pledgeinsize(codec, -1, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, Int64(-1), e) === :ok @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok TranscodingStreams.finalize(codec) end @@ -267,7 +267,7 @@ include("utils.jl") GC.@preserve data let m = TranscodingStreams.Memory(pointer(data), length(data)) if isdefined(TranscodingStreams, :pledgeinsize) try - TranscodingStreams.pledgeinsize(codec, 10, TranscodingStreams.Error()) + TranscodingStreams.pledgeinsize(codec, Int64(10), TranscodingStreams.Error()) catch end end From 0e13c103bb2afea624e5943c9cd672773a21d4ed Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sat, 5 Oct 2024 20:39:34 -0400 Subject: [PATCH 7/8] remove dead code --- src/compression.jl | 9 ++------- src/libzstd.jl | 6 +++++- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/compression.jl b/src/compression.jl index 6a2b7b8..8ce79dd 100644 --- a/src/compression.jl +++ b/src/compression.jl @@ -101,13 +101,8 @@ function TranscodingStreams.startproc(codec::ZstdCompressor, mode::Symbol, error return :error end end - code = reset!(codec.cstream) - if iserror(code) - error[] = ErrorException("zstd error resetting compression context") - :error - else - :ok - end + reset!(codec.cstream) + return :ok end if isdefined(TranscodingStreams, :pledgeinsize) diff --git a/src/libzstd.jl b/src/libzstd.jl index c17f69d..dbbbb4b 100644 --- a/src/libzstd.jl +++ b/src/libzstd.jl @@ -60,7 +60,11 @@ function reset!(cstream::CStream) res = LibZstd.ZSTD_CCtx_reset(cstream, LibZstd.ZSTD_reset_session_only) reset!(cstream.ibuffer) reset!(cstream.obuffer) - return res + if iserror(res) + # According to zstd.h "Resetting session never fails" so this branch should be unreachable. + error("unreachable") + end + return end """ From 055af7b090a26a64bcae80c6bb6592a829d5aeaf Mon Sep 17 00:00:00 2001 From: nhz2 Date: Sun, 6 Oct 2024 14:57:58 -0400 Subject: [PATCH 8/8] bump compat --- Project.toml | 4 +- src/compression.jl | 32 ++++++------ test/runtests.jl | 118 ++++++++++++++++++++++----------------------- 3 files changed, 74 insertions(+), 80 deletions(-) diff --git a/Project.toml b/Project.toml index fa3a41a..15efc4f 100644 --- a/Project.toml +++ b/Project.toml @@ -10,6 +10,6 @@ TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" [compat] -TranscodingStreams = "0.9, 0.10, 0.11" +TranscodingStreams = "0.11.3" Zstd_jll = "1.5.5" -julia = "1.3" +julia = "1.6" diff --git a/src/compression.jl b/src/compression.jl index 8ce79dd..7476650 100644 --- a/src/compression.jl +++ b/src/compression.jl @@ -105,23 +105,21 @@ function TranscodingStreams.startproc(codec::ZstdCompressor, mode::Symbol, error return :ok end -if isdefined(TranscodingStreams, :pledgeinsize) - function TranscodingStreams.pledgeinsize(codec::ZstdCompressor, insize::Int64, error::Error)::Symbol - if codec.cstream.ptr == C_NULL - Base.error("`startproc` must be called before `pledgeinsize`") - end - srcsize = if signbit(insize) - ZSTD_CONTENTSIZE_UNKNOWN - else - Culonglong(insize) - end - code = LibZstd.ZSTD_CCtx_setPledgedSrcSize(codec.cstream, srcsize) - if iserror(code) - error[] = ErrorException("zstd error setting pledged source size") - :error - else - :ok - end +function TranscodingStreams.pledgeinsize(codec::ZstdCompressor, insize::Int64, error::Error)::Symbol + if codec.cstream.ptr == C_NULL + Base.error("`startproc` must be called before `pledgeinsize`") + end + srcsize = if signbit(insize) + ZSTD_CONTENTSIZE_UNKNOWN + else + Culonglong(insize) + end + code = LibZstd.ZSTD_CCtx_setPledgedSrcSize(codec.cstream, srcsize) + if iserror(code) + error[] = ErrorException("zstd error setting pledged source size") + :error + else + :ok end end diff --git a/test/runtests.jl b/test/runtests.jl index 78402cc..56b8d5a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -165,63 +165,61 @@ include("utils.jl") end @testset "pledgeinsize" begin - if isdefined(TranscodingStreams, :pledgeinsize) - # when pledgeinsize is available transcode should save the - # decompressed size in a header - for n in [0:30; 1000; 1000000;] - v = transcode(ZstdCompressor, rand(UInt8, n)) - @test CodecZstd.find_decompressed_size(v) == n - end + # when pledgeinsize is available transcode should save the + # decompressed size in a header + for n in [0:30; 1000; 1000000;] + v = transcode(ZstdCompressor, rand(UInt8, n)) + @test CodecZstd.find_decompressed_size(v) == n + end - # Test what happens if pledgeinsize promise is broken - d1 = zeros(UInt8, 10000) - d2 = zeros(UInt8, 10000) - GC.@preserve d1 d2 begin - @testset "too many bytes" begin - m1 = TranscodingStreams.Memory(pointer(d1), 1000) - m2 = TranscodingStreams.Memory(pointer(d2), 1000) - codec = ZstdCompressor() - e = TranscodingStreams.Error() - @test TranscodingStreams.startproc(codec, :read, e) === :ok - @test TranscodingStreams.pledgeinsize(codec, Int64(10), e) === :ok - @test TranscodingStreams.process(codec, m1, m2, e) === (0, 0, :error) - @test e[] == ErrorException("zstd error: Src size is incorrect") - TranscodingStreams.finalize(codec) - end - @testset "too few bytes" begin - m1 = TranscodingStreams.Memory(pointer(d1), 10) - m2 = TranscodingStreams.Memory(pointer(d2), 1000) - codec = ZstdCompressor() - e = TranscodingStreams.Error() - @test TranscodingStreams.startproc(codec, :read, e) === :ok - @test TranscodingStreams.pledgeinsize(codec, Int64(10000), e) === :ok - @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok - m1 = TranscodingStreams.Memory(pointer(d1), 0) - @test TranscodingStreams.process(codec, m1, m2, e)[3] === :error - @test e[] == ErrorException("zstd error: Src size is incorrect") - TranscodingStreams.finalize(codec) - end - @testset "set pledgeinsize after process" begin - m1 = TranscodingStreams.Memory(pointer(d1), 1000) - m2 = TranscodingStreams.Memory(pointer(d2), 1000) - codec = ZstdCompressor() - e = TranscodingStreams.Error() - @test TranscodingStreams.startproc(codec, :read, e) === :ok - @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok - @test TranscodingStreams.pledgeinsize(codec, Int64(10000), e) === :error - @test e[] == ErrorException("zstd error setting pledged source size") - TranscodingStreams.finalize(codec) - end - @testset "set unknown pledgeinsize" begin - m1 = TranscodingStreams.Memory(pointer(d1), 1000) - m2 = TranscodingStreams.Memory(pointer(d2), 1000) - codec = ZstdCompressor() - e = TranscodingStreams.Error() - @test TranscodingStreams.startproc(codec, :read, e) === :ok - @test TranscodingStreams.pledgeinsize(codec, Int64(-1), e) === :ok - @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok - TranscodingStreams.finalize(codec) - end + # Test what happens if pledgeinsize promise is broken + d1 = zeros(UInt8, 10000) + d2 = zeros(UInt8, 10000) + GC.@preserve d1 d2 begin + @testset "too many bytes" begin + m1 = TranscodingStreams.Memory(pointer(d1), 1000) + m2 = TranscodingStreams.Memory(pointer(d2), 1000) + codec = ZstdCompressor() + e = TranscodingStreams.Error() + @test TranscodingStreams.startproc(codec, :read, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, Int64(10), e) === :ok + @test TranscodingStreams.process(codec, m1, m2, e) === (0, 0, :error) + @test e[] == ErrorException("zstd error: Src size is incorrect") + TranscodingStreams.finalize(codec) + end + @testset "too few bytes" begin + m1 = TranscodingStreams.Memory(pointer(d1), 10) + m2 = TranscodingStreams.Memory(pointer(d2), 1000) + codec = ZstdCompressor() + e = TranscodingStreams.Error() + @test TranscodingStreams.startproc(codec, :read, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, Int64(10000), e) === :ok + @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok + m1 = TranscodingStreams.Memory(pointer(d1), 0) + @test TranscodingStreams.process(codec, m1, m2, e)[3] === :error + @test e[] == ErrorException("zstd error: Src size is incorrect") + TranscodingStreams.finalize(codec) + end + @testset "set pledgeinsize after process" begin + m1 = TranscodingStreams.Memory(pointer(d1), 1000) + m2 = TranscodingStreams.Memory(pointer(d2), 1000) + codec = ZstdCompressor() + e = TranscodingStreams.Error() + @test TranscodingStreams.startproc(codec, :read, e) === :ok + @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok + @test TranscodingStreams.pledgeinsize(codec, Int64(10000), e) === :error + @test e[] == ErrorException("zstd error setting pledged source size") + TranscodingStreams.finalize(codec) + end + @testset "set unknown pledgeinsize" begin + m1 = TranscodingStreams.Memory(pointer(d1), 1000) + m2 = TranscodingStreams.Memory(pointer(d2), 1000) + codec = ZstdCompressor() + e = TranscodingStreams.Error() + @test TranscodingStreams.startproc(codec, :read, e) === :ok + @test TranscodingStreams.pledgeinsize(codec, Int64(-1), e) === :ok + @test TranscodingStreams.process(codec, m1, m2, e)[3] === :ok + TranscodingStreams.finalize(codec) end end end @@ -265,11 +263,9 @@ include("utils.jl") TranscodingStreams.finalize(codec) data = [0x00,0x01] GC.@preserve data let m = TranscodingStreams.Memory(pointer(data), length(data)) - if isdefined(TranscodingStreams, :pledgeinsize) - try - TranscodingStreams.pledgeinsize(codec, Int64(10), TranscodingStreams.Error()) - catch - end + try + TranscodingStreams.pledgeinsize(codec, Int64(10), TranscodingStreams.Error()) + catch end try TranscodingStreams.expectedsize(codec, m)