From 76f3e9f56fbad9f4520fa2d512291dd56c6e6cb5 Mon Sep 17 00:00:00 2001 From: Samuel Li Date: Wed, 14 Dec 2022 18:03:15 -0700 Subject: [PATCH] Header keeps bits (#182) * header keeps the number of useful bits rather than bytes. Fix issue #181 * compressor_3d releases compressor resources before doing decompression * bump version to 0.5.2 Co-authored-by: Samuel Li --- CMakeLists.txt | 2 +- SperrConfig.h.in | 1 + src/SPECK2D.cpp | 4 --- src/SPECK3D.cpp | 5 ---- src/SPECK_Storage.cpp | 48 ++++++++++++++++++++------------- utilities/compressor_3d.cpp | 54 +++++++++++++++++++------------------ utilities/show_version.cpp | 3 ++- 7 files changed, 61 insertions(+), 56 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 54dd7609..b780506b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 3.14) -project(SPERR VERSION 0.5.1 DESCRIPTION "Lossy Scientific Compression with SPERR") +project(SPERR VERSION 0.5.2 DESCRIPTION "Lossy Scientific Compression with SPERR") # # specify the C++ standard diff --git a/SperrConfig.h.in b/SperrConfig.h.in index a037f409..1af89c96 100644 --- a/SperrConfig.h.in +++ b/SperrConfig.h.in @@ -3,6 +3,7 @@ #define SPERR_VERSION_MAJOR @SPERR_VERSION_MAJOR@ #define SPERR_VERSION_MINOR @SPERR_VERSION_MINOR@ +#define SPERR_VERSION_PATCH @SPERR_VERSION_PATCH@ static const char* SPERR_GIT_SHA1 = "@GIT_SHA1@"; static const char* SPERR_GIT_BRANCH = "@GIT_BRANCH@"; diff --git a/src/SPECK2D.cpp b/src/SPECK2D.cpp index 848044ce..a2d4930b 100644 --- a/src/SPECK2D.cpp +++ b/src/SPECK2D.cpp @@ -107,10 +107,6 @@ auto sperr::SPECK2D::encode() -> RTNType m_clean_LIS(); } - // Fill the bit buffer to multiplies of eight - while (m_bit_buffer.size() % 8 != 0) - m_bit_buffer.push_back(false); - // Finally we prepare the bitstream rtn = m_prepare_encoded_bitstream(); return rtn; diff --git a/src/SPECK3D.cpp b/src/SPECK3D.cpp index fa7f7881..efcb763a 100644 --- a/src/SPECK3D.cpp +++ b/src/SPECK3D.cpp @@ -117,11 +117,6 @@ auto sperr::SPECK3D::encode() -> RTNType m_clean_LIS(); } - // If the bit buffer has the last byte half-empty, let's fill in zero's. - // The decoding process will not read them anyway. - while (m_bit_buffer.size() % 8 != 0) - m_bit_buffer.push_back(false); - // Finally we prepare the bitstream rtn = m_prepare_encoded_bitstream(); diff --git a/src/SPECK_Storage.cpp b/src/SPECK_Storage.cpp index 0e3de2f8..f15aad14 100644 --- a/src/SPECK_Storage.cpp +++ b/src/SPECK_Storage.cpp @@ -68,26 +68,31 @@ void sperr::SPECK_Storage::set_data_range(double range) auto sperr::SPECK_Storage::m_prepare_encoded_bitstream() -> RTNType { // Header definition: 12 bytes in total: - // m_max_threshold_f, stream_len + // m_max_threshold_f, num_useful_bits // float, uint64_t - assert(m_bit_buffer.size() % 8 == 0); - const uint64_t bit_in_byte = m_bit_buffer.size() / 8; - const size_t total_size = m_header_size + bit_in_byte; + const uint64_t useful_bits = m_bit_buffer.size(); + while (m_bit_buffer.size() % 8 != 0) + m_bit_buffer.push_back(false); + const size_t total_size = m_header_size + m_bit_buffer.size() / 8; m_encoded_stream.resize(total_size); - auto* const ptr = m_encoded_stream.data(); // Fill header size_t pos = 0; + auto* const ptr = m_encoded_stream.data(); std::memcpy(ptr + pos, &m_max_threshold_f, sizeof(m_max_threshold_f)); pos += sizeof(m_max_threshold_f); - std::memcpy(ptr + pos, &bit_in_byte, sizeof(bit_in_byte)); - pos += sizeof(bit_in_byte); + std::memcpy(ptr + pos, &useful_bits, sizeof(useful_bits)); + pos += sizeof(useful_bits); assert(pos == m_header_size); // Assemble the bitstream into bytes auto rtn = sperr::pack_booleans(m_encoded_stream, m_bit_buffer, pos); + + // Restore the number of useful bits + m_bit_buffer.resize(useful_bits); + return rtn; } @@ -105,20 +110,21 @@ auto sperr::SPECK_Storage::parse_encoded_bitstream(const void* comp_buf, size_t std::memcpy(&m_max_threshold_f, ptr + pos, sizeof(m_max_threshold_f)); pos += sizeof(m_max_threshold_f); - uint64_t bit_in_byte = 0; - std::memcpy(&bit_in_byte, ptr + pos, sizeof(bit_in_byte)); - pos += sizeof(bit_in_byte); - if (bit_in_byte != comp_size - pos) + uint64_t useful_bits = 0; + std::memcpy(&useful_bits, ptr + pos, sizeof(useful_bits)); + pos += sizeof(useful_bits); + auto num_bytes = useful_bits / 8; + if (useful_bits % 8 != 0) + ++num_bytes; + if (num_bytes != comp_size - pos) return RTNType::BitstreamWrongLen; // Unpack bits - const auto num_of_bools = (comp_size - pos) * 8; - m_bit_buffer.resize(num_of_bools); // allocate enough space before passing it around + m_bit_buffer.resize(num_bytes * 8); // Allocate enough space before passing it around auto rtn = sperr::unpack_booleans(m_bit_buffer, comp_buf, comp_size, pos); - if (rtn != RTNType::Good) - return rtn; + m_bit_buffer.resize(useful_bits); // Restore the actual useful number of bits - return RTNType::Good; + return rtn; } auto sperr::SPECK_Storage::view_encoded_bitstream() const -> const vec8_type& @@ -137,10 +143,14 @@ auto sperr::SPECK_Storage::get_speck_stream_size(const void* buf) const -> uint6 // go retrieve the value stored in the last 8 bytes of the header // const uint8_t* const ptr = static_cast(buf); - uint64_t bit_in_byte; - std::memcpy(&bit_in_byte, ptr + m_header_size - 8, sizeof(bit_in_byte)); + uint64_t useful_bits; + std::memcpy(&useful_bits, ptr + m_header_size - 8, sizeof(useful_bits)); + + auto num_bytes = useful_bits / 8; + if (useful_bits % 8 != 0) + ++num_bytes; - return (m_header_size + size_t(bit_in_byte)); + return (m_header_size + num_bytes); } void sperr::SPECK_Storage::set_dimensions(dims_type dims) diff --git a/utilities/compressor_3d.cpp b/utilities/compressor_3d.cpp index 7dd5989b..f5373328 100644 --- a/utilities/compressor_3d.cpp +++ b/utilities/compressor_3d.cpp @@ -7,6 +7,7 @@ #include #include #include +#include int main(int argc, char* argv[]) { @@ -80,26 +81,26 @@ int main(int argc, char* argv[]) std::cout << "Compression mode is unclear. Did you give one and only one " "compression specification?" << std::endl; - return 1; + return __LINE__; } // Do some sanity check on compression parameters if (mode == sperr::CompMode::FixedSize) { if (bpp <= 0.0 || bpp >= 64.0) { std::cout << "Bit-per-pixel value must be between 0.0 and 64.0!" << std::endl; - return 1; + return __LINE__; } } else if (mode == sperr::CompMode::FixedPSNR) { if (psnr <= 0.0) { std::cout << "Target PSNR must be positive!" << std::endl; - return 1; + return __LINE__; } } else if (mode == sperr::CompMode::FixedPWE) { if (pwe <= 0.0) { std::cout << "Target PWE must be positive!" << std::endl; - return 1; + return __LINE__; } } @@ -109,24 +110,24 @@ int main(int argc, char* argv[]) if ((use_double && orig.size() != total_vals * sizeof(double)) || (!use_double && orig.size() != total_vals * sizeof(float))) { std::cerr << "Read input file error: " << input_file << std::endl; - return 1; + return __LINE__; } // Use a compressor and take the input data - SPERR3D_OMP_C compressor; - compressor.set_num_threads(omp_num_threads); + auto compressor = std::make_unique(); + compressor->set_num_threads(omp_num_threads); auto rtn = sperr::RTNType::Good; if (use_double) { - rtn = compressor.copy_data(reinterpret_cast(orig.data()), total_vals, - {dims[0], dims[1], dims[2]}, {chunks[0], chunks[1], chunks[2]}); + rtn = compressor->copy_data(reinterpret_cast(orig.data()), total_vals, + {dims[0], dims[1], dims[2]}, {chunks[0], chunks[1], chunks[2]}); } else { - rtn = compressor.copy_data(reinterpret_cast(orig.data()), total_vals, - {dims[0], dims[1], dims[2]}, {chunks[0], chunks[1], chunks[2]}); + rtn = compressor->copy_data(reinterpret_cast(orig.data()), total_vals, + {dims[0], dims[1], dims[2]}, {chunks[0], chunks[1], chunks[2]}); } if (rtn != sperr::RTNType::Good) { std::cerr << "Copy data failed!" << std::endl; - return 1; + return __LINE__; } // Free up memory if we don't need to compute stats @@ -138,38 +139,38 @@ int main(int argc, char* argv[]) // Tell the compressor which compression mode to use. switch (mode) { case sperr::CompMode::FixedSize: - rtn = compressor.set_target_bpp(bpp); + rtn = compressor->set_target_bpp(bpp); break; case sperr::CompMode::FixedPSNR: - compressor.set_target_psnr(psnr); + compressor->set_target_psnr(psnr); break; default: - compressor.set_target_pwe(pwe); + compressor->set_target_pwe(pwe); break; } if (rtn != sperr::RTNType::Good) { std::cerr << "Set bit-per-pixel failed!" << std::endl; - return 1; + return __LINE__; } // Perform the actual compression - rtn = compressor.compress(); + rtn = compressor->compress(); switch (rtn) { case sperr::RTNType::QzLevelTooBig: std::cerr << "Compression failed because `qz` is set too big!" << std::endl; - return 1; + return __LINE__; case sperr::RTNType::Good: break; default: std::cerr << "Compression failed!" << std::endl; - return 1; + return __LINE__; } // Get a hold of the encoded bitstream. - auto stream = compressor.get_encoded_bitstream(); + auto stream = compressor->get_encoded_bitstream(); if (stream.empty()) { std::cerr << "Compression bitstream empty!" << std::endl; - return 1; + return __LINE__; } // Write out the encoded bitstream. @@ -177,12 +178,15 @@ int main(int argc, char* argv[]) rtn = sperr::write_n_bytes(output_file, stream.size(), stream.data()); if (rtn != sperr::RTNType::Good) { std::cerr << "Write compressed file failed!" << std::endl; - return 1; + return __LINE__; } } // Calculate and print statistics if (show_stats) { + const auto out_stats = compressor->get_outlier_stats(); + compressor.reset(nullptr); + const auto bpp = stream.size() * 8.0 / total_vals; // Use a decompressor to decompress and collect error statistics @@ -190,11 +194,11 @@ int main(int argc, char* argv[]) decompressor.set_num_threads(omp_num_threads); rtn = decompressor.use_bitstream(stream.data(), stream.size()); if (rtn != RTNType::Good) - return 1; + return __LINE__; rtn = decompressor.decompress(stream.data()); if (rtn != RTNType::Good) - return 1; + return __LINE__; if (use_double) { const auto& recover = decompressor.view_data(); @@ -226,8 +230,6 @@ int main(int argc, char* argv[]) } if (mode == sperr::CompMode::FixedPWE) { - // Also collect outlier statistics - const auto out_stats = compressor.get_outlier_stats(); if (out_stats.first == 0) { std::cout << "There were no outliers corrected!\n"; } diff --git a/utilities/show_version.cpp b/utilities/show_version.cpp index 6e20852d..cd894588 100644 --- a/utilities/show_version.cpp +++ b/utilities/show_version.cpp @@ -4,7 +4,8 @@ int main() { - std::cout << "SPERR version: " << SPERR_VERSION_MAJOR << "." << SPERR_VERSION_MINOR << std::endl; + std::cout << "SPERR version: " << SPERR_VERSION_MAJOR << "." << SPERR_VERSION_MINOR << "." + << SPERR_VERSION_PATCH << std::endl; std::cout << "Based on code Branch: " << SPERR_GIT_BRANCH << std::endl; std::cout << "Based on code SHA1 : " << SPERR_GIT_SHA1 << std::endl; }