Merge branch 'master' into nz-pledgeinsize

.gitignore

@@ -2,13 +2,13 @@
 *.jl.*.cov
 *.jl.mem
 /Manifest.toml
-/deps/build.log
-/deps/deps.jl
-/deps/usr/
 src/libzstd/LibTemplate.jl
 src/libzstd/ctypes.jl
 /github/
 settings.json
+**/*.json
+**/*.json.tmp
+benchmark/Manifest.toml
 dictionary
 github_users_sample_set.tar
 github_users_sample_set.tar.zst

README.md

@@ -1,7 +1,7 @@
 CodecZstd.jl
 ============
 
-[![codecov.io][codecov-img]][codecov-url]
+[![codecov](https://codecov.io/gh/JuliaIO/CodecZstd.jl/graph/badge.svg?token=CgSrhnEKdy)](https://codecov.io/gh/JuliaIO/CodecZstd.jl)
 
 ## Installation
 
@@ -46,11 +46,4 @@ This package exports following codecs and streams:
 
 Version 0.8.3 also introduced the virtual codec `ZstdFrameCompressor` which stores the decompressed content size in the frame header. Currently, `ZstdFrameCompressor` is an alternate constructor for `ZstdCompressor`, but that is an implementation detail which should not be relied upon.
 
-See docstrings and [TranscodingStreams.jl](https://github.com/bicycle1885/TranscodingStreams.jl) for details.
-
-[travisci-img]: https://travis-ci.org/bicycle1885/CodecZstd.jl.svg?branch=master
-[travisci-url]: https://travis-ci.org/bicycle1885/CodecZstd.jl
-[appveyor-img]: https://ci.appveyor.com/api/projects/status/u58v32yenqf19x2a?svg=true
-[appveyor-url]: https://ci.appveyor.com/project/bicycle1885/codeczstd-jl
-[codecov-img]: http://codecov.io/github/bicycle1885/CodecZstd.jl/coverage.svg?branch=master
-[codecov-url]: http://codecov.io/github/bicycle1885/CodecZstd.jl?branch=master
+See docstrings and [TranscodingStreams.jl](https://github.com/JuliaIO/TranscodingStreams.jl) for details.

benchmark/Project.toml

@@ -0,0 +1,6 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"

benchmark/README.md

@@ -0,0 +1,11 @@
+# CodecZstd.jl benchmarks
+
+This directory contains benchmarks for CodecZstd. To run all the
+benchmarks, launch `julia --project=benchmark` and enter:
+
+``` julia
+using PkgBenchmark
+import CodecZstd
+
+benchmarkpkg(CodecZstd)
+```

benchmark/benchmarks.jl

@@ -0,0 +1,32 @@
+using BenchmarkTools
+using Random
+using TranscodingStreams
+using CodecZstd
+
+const SUITE = BenchmarkGroup()
+cbench = SUITE["compression"] = BenchmarkGroup()
+dbench = SUITE["decompression"] = BenchmarkGroup()
+
+ccodec = ZstdCompressor()
+dcodec = ZstdDecompressor()
+TranscodingStreams.initialize(ccodec)
+TranscodingStreams.initialize(dcodec)
+
+for N in [100, 100000, 1000000]
+ u1 = rand(Xoshiro(1234), UInt8, N)
+ c1 = transcode(ZstdCompressor, u1)
+ cbench["uncompressible"][N] = @benchmarkable transcode($ccodec, $u1)
+ dbench["uncompressible"][N] = @benchmarkable transcode($dcodec, $c1)
+
+ u2 = rand(Xoshiro(1234), 0x00:0x01, N)
+ c2 = transcode(ZstdCompressor, u2)
+ cbench["compressible-bytes"][N] = @benchmarkable transcode($ccodec, $u2)
+ dbench["compressible-bytes"][N] = @benchmarkable transcode($dcodec, $c2)
+
+ f = round.(randn(Xoshiro(1234), N); base=2, digits=7)
+ # byte shuffle
+ u3 = vec(permutedims(reinterpret(reshape, UInt8, f),(2,1)))
+ c3 = transcode(ZstdCompressor, u3)
+ cbench["byteshuffle"][N] = @benchmarkable transcode($ccodec, $u3)
+ dbench["byteshuffle"][N] = @benchmarkable transcode($dcodec, $c3)
+end

src/compression.jl

@@ -78,16 +78,6 @@ end
 # Methods
 # -------
 
-function TranscodingStreams.initialize(codec::ZstdCompressor)
- code = initialize!(codec.cstream, codec.level)
- if iserror(code)
- zstderror(codec.cstream, code)
- end
- reset!(codec.cstream.ibuffer)
- reset!(codec.cstream.obuffer)
- return
-end
-
 function TranscodingStreams.finalize(codec::ZstdCompressor)
  if codec.cstream.ptr != C_NULL
  code = free!(codec.cstream)
@@ -96,12 +86,21 @@ function TranscodingStreams.finalize(codec::ZstdCompressor)
  end
  codec.cstream.ptr = C_NULL
  end
- reset!(codec.cstream.ibuffer)
- reset!(codec.cstream.obuffer)
  return
 end
 
-function TranscodingStreams.startproc(codec::ZstdCompressor, mode::Symbol, error::Error)::Symbol
+function TranscodingStreams.startproc(codec::ZstdCompressor, mode::Symbol, error::Error)
+ if codec.cstream.ptr == C_NULL
+ codec.cstream.ptr = LibZstd.ZSTD_createCStream()
+ if codec.cstream.ptr == C_NULL
+ throw(OutOfMemoryError())
+ end
+ i_code = initialize!(codec.cstream, codec.level)
+ if iserror(i_code)
+ error[] = ErrorException("zstd initialization error")
+ return :error
+ end
+ end
  code = reset!(codec.cstream)
  if iserror(code)
  error[] = ErrorException("zstd error resetting compression context")
@@ -129,6 +128,9 @@ if isdefined(TranscodingStreams, :pledgeinsize)
 end
 
 function TranscodingStreams.process(codec::ZstdCompressor, input::Memory, output::Memory, error::Error)
+ if codec.cstream.ptr == C_NULL
+ error("startproc must be called before process")
+ end
  cstream = codec.cstream
  ibuffer_starting_pos = UInt(0)
  if codec.endOp == LibZstd.ZSTD_e_end &&

src/decompression.jl

@@ -33,16 +33,6 @@ end
 # Methods
 # -------
 
-function TranscodingStreams.initialize(codec::ZstdDecompressor)
- code = initialize!(codec.dstream)
- if iserror(code)
- zstderror(codec.dstream, code)
- end
- reset!(codec.dstream.ibuffer)
- reset!(codec.dstream.obuffer)
- return
-end
-
 function TranscodingStreams.finalize(codec::ZstdDecompressor)
  if codec.dstream.ptr != C_NULL
  code = free!(codec.dstream)
@@ -51,12 +41,21 @@ function TranscodingStreams.finalize(codec::ZstdDecompressor)
  end
  codec.dstream.ptr = C_NULL
  end
- reset!(codec.dstream.ibuffer)
- reset!(codec.dstream.obuffer)
  return
 end
 
 function TranscodingStreams.startproc(codec::ZstdDecompressor, mode::Symbol, error::Error)
+ if codec.dstream.ptr == C_NULL
+ codec.dstream.ptr = LibZstd.ZSTD_createDStream()
+ if codec.dstream.ptr == C_NULL
+ throw(OutOfMemoryError())
+ end
+ i_code = initialize!(codec.dstream)
+ if iserror(i_code)
+ error[] = ErrorException("zstd initialization error")
+ return :error
+ end
+ end
  code = reset!(codec.dstream)
  if iserror(code)
  error[] = ErrorException("zstd error")
@@ -66,6 +65,9 @@ function TranscodingStreams.startproc(codec::ZstdDecompressor, mode::Symbol, err
 end
 
 function TranscodingStreams.process(codec::ZstdDecompressor, input::Memory, output::Memory, error::Error)
+ if codec.dstream.ptr == C_NULL
+ error("startproc must be called before process")
+ end
  dstream = codec.dstream
  dstream.ibuffer.src = input.ptr
  dstream.ibuffer.size = input.size

src/libzstd.jl

@@ -44,11 +44,7 @@ mutable struct CStream
  obuffer::OutBuffer
 
  function CStream()
- ptr = LibZstd.ZSTD_createCStream()
- if ptr == C_NULL
- throw(OutOfMemoryError())
- end
- return new(ptr, InBuffer(), OutBuffer())
+ return new(C_NULL, InBuffer(), OutBuffer())
  end
 end
 
@@ -115,11 +111,7 @@ mutable struct DStream
  obuffer::OutBuffer
 
  function DStream()
- ptr = LibZstd.ZSTD_createDStream()
- if ptr == C_NULL
- throw(OutOfMemoryError())
- end
- return new(ptr, InBuffer(), OutBuffer())
+ return new(C_NULL, InBuffer(), OutBuffer())
  end
 end
 Base.unsafe_convert(::Type{Ptr{LibZstd.ZSTD_DStream}}, dstream::DStream) = dstream.ptr
@@ -133,6 +125,8 @@ end
 function reset!(dstream::DStream)
  # LibZstd.ZSTD_resetDStream is deprecated
  # https://github.com/facebook/zstd/blob/9d2a45a705e22ad4817b41442949cd0f78597154/lib/zstd.h#L2332-L2339
+ reset!(dstream.ibuffer)
+ reset!(dstream.obuffer)
  return LibZstd.ZSTD_DCtx_reset(dstream, LibZstd.ZSTD_reset_session_only)
 end
 

test/compress_endOp.jl

@@ -3,27 +3,29 @@ using Test
 
 @testset "compress! endOp = :continue" begin
  data = rand(1:100, 1024*1024)
- cstream = CodecZstd.CStream()
- cstream.ibuffer.src = pointer(data)
- cstream.ibuffer.size = sizeof(data)
- cstream.ibuffer.pos = 0
- cstream.obuffer.dst = Base.Libc.malloc(sizeof(data)*2)
- cstream.obuffer.size = sizeof(data)*2
- cstream.obuffer.pos = 0
- try
- GC.@preserve data begin
+ GC.@preserve data begin
+ cstream = CodecZstd.CStream()
+ cstream.ptr = CodecZstd.LibZstd.ZSTD_createCStream()
+ cstream.ibuffer.src = pointer(data)
+ cstream.ibuffer.size = sizeof(data)
+ cstream.ibuffer.pos = 0
+ cstream.obuffer.dst = Base.Libc.malloc(sizeof(data)*2)
+ cstream.obuffer.size = sizeof(data)*2
+ cstream.obuffer.pos = 0
+ try
  # default endOp
  @test CodecZstd.compress!(cstream; endOp=:continue) == 0
  @test CodecZstd.find_decompressed_size(cstream.obuffer.dst, cstream.obuffer.pos) == CodecZstd.ZSTD_CONTENTSIZE_UNKNOWN
+ finally
+ Base.Libc.free(cstream.obuffer.dst)
  end
- finally
- Base.Libc.free(cstream.obuffer.dst)
  end
 end
 
 @testset "compress! endOp = :flush" begin
  data = rand(1:100, 1024*1024)
  cstream = CodecZstd.CStream()
+ cstream.ptr = CodecZstd.LibZstd.ZSTD_createCStream()
  cstream.ibuffer.src = pointer(data)
  cstream.ibuffer.size = sizeof(data)
  cstream.ibuffer.pos = 0
@@ -43,6 +45,7 @@ end
 @testset "compress! endOp = :end" begin
  data = rand(1:100, 1024*1024)
  cstream = CodecZstd.CStream()
+ cstream.ptr = CodecZstd.LibZstd.ZSTD_createCStream()
  cstream.ibuffer.src = pointer(data)
  cstream.ibuffer.size = sizeof(data)
  cstream.ibuffer.pos = 0