Header keeps bits (#182)

* header keeps the number of useful bits rather than bytes. Fix issue #181

* compressor_3d releases compressor resources before doing decompression

* bump version to 0.5.2

Co-authored-by: Samuel Li <Sam@Navada>

CMakeLists.txt

@@ -2,7 +2,7 @@
 
 cmake_minimum_required(VERSION 3.14)
 
-project(SPERR VERSION 0.5.1 DESCRIPTION "Lossy Scientific Compression with SPERR")
+project(SPERR VERSION 0.5.2 DESCRIPTION "Lossy Scientific Compression with SPERR")
 
 #
 # specify the C++ standard

SperrConfig.h.in

@@ -3,6 +3,7 @@
 
 #define SPERR_VERSION_MAJOR @SPERR_VERSION_MAJOR@
 #define SPERR_VERSION_MINOR @SPERR_VERSION_MINOR@
+#define SPERR_VERSION_PATCH @SPERR_VERSION_PATCH@
 
 static const char* SPERR_GIT_SHA1 = "@GIT_SHA1@";
 static const char* SPERR_GIT_BRANCH = "@GIT_BRANCH@";

src/SPECK2D.cpp

@@ -107,10 +107,6 @@ auto sperr::SPECK2D::encode() -> RTNType
  m_clean_LIS();
  }
 
- // Fill the bit buffer to multiplies of eight
- while (m_bit_buffer.size() % 8 != 0)
- m_bit_buffer.push_back(false);
-
  // Finally we prepare the bitstream
  rtn = m_prepare_encoded_bitstream();
  return rtn;

src/SPECK3D.cpp

@@ -117,11 +117,6 @@ auto sperr::SPECK3D::encode() -> RTNType
  m_clean_LIS();
  }
 
- // If the bit buffer has the last byte half-empty, let's fill in zero's.
- // The decoding process will not read them anyway.
- while (m_bit_buffer.size() % 8 != 0)
- m_bit_buffer.push_back(false);
-
  // Finally we prepare the bitstream
  rtn = m_prepare_encoded_bitstream();
 

src/SPECK_Storage.cpp

@@ -68,26 +68,31 @@ void sperr::SPECK_Storage::set_data_range(double range)
 auto sperr::SPECK_Storage::m_prepare_encoded_bitstream() -> RTNType
 {
  // Header definition: 12 bytes in total:
- // m_max_threshold_f, stream_len
+ // m_max_threshold_f, num_useful_bits
  // float, uint64_t
 
- assert(m_bit_buffer.size() % 8 == 0);
- const uint64_t bit_in_byte = m_bit_buffer.size() / 8;
- const size_t total_size = m_header_size + bit_in_byte;
+ const uint64_t useful_bits = m_bit_buffer.size();
+ while (m_bit_buffer.size() % 8 != 0)
+ m_bit_buffer.push_back(false);
+ const size_t total_size = m_header_size + m_bit_buffer.size() / 8;
  m_encoded_stream.resize(total_size);
- auto* const ptr = m_encoded_stream.data();
 
  // Fill header
  size_t pos = 0;
+ auto* const ptr = m_encoded_stream.data();
  std::memcpy(ptr + pos, &m_max_threshold_f, sizeof(m_max_threshold_f));
  pos += sizeof(m_max_threshold_f);
 
- std::memcpy(ptr + pos, &bit_in_byte, sizeof(bit_in_byte));
- pos += sizeof(bit_in_byte);
+ std::memcpy(ptr + pos, &useful_bits, sizeof(useful_bits));
+ pos += sizeof(useful_bits);
  assert(pos == m_header_size);
 
  // Assemble the bitstream into bytes
  auto rtn = sperr::pack_booleans(m_encoded_stream, m_bit_buffer, pos);
+
+ // Restore the number of useful bits
+ m_bit_buffer.resize(useful_bits);
+
  return rtn;
 }
 
@@ -105,20 +110,21 @@ auto sperr::SPECK_Storage::parse_encoded_bitstream(const void* comp_buf, size_t
  std::memcpy(&m_max_threshold_f, ptr + pos, sizeof(m_max_threshold_f));
  pos += sizeof(m_max_threshold_f);
 
- uint64_t bit_in_byte = 0;
- std::memcpy(&bit_in_byte, ptr + pos, sizeof(bit_in_byte));
- pos += sizeof(bit_in_byte);
- if (bit_in_byte != comp_size - pos)
+ uint64_t useful_bits = 0;
+ std::memcpy(&useful_bits, ptr + pos, sizeof(useful_bits));
+ pos += sizeof(useful_bits);
+ auto num_bytes = useful_bits / 8;
+ if (useful_bits % 8 != 0)
+ ++num_bytes;
+ if (num_bytes != comp_size - pos)
  return RTNType::BitstreamWrongLen;
 
  // Unpack bits
- const auto num_of_bools = (comp_size - pos) * 8;
- m_bit_buffer.resize(num_of_bools); // allocate enough space before passing it around
+ m_bit_buffer.resize(num_bytes * 8); // Allocate enough space before passing it around
  auto rtn = sperr::unpack_booleans(m_bit_buffer, comp_buf, comp_size, pos);
- if (rtn != RTNType::Good)
- return rtn;
+ m_bit_buffer.resize(useful_bits); // Restore the actual useful number of bits
 
- return RTNType::Good;
+ return rtn;
 }
 
 auto sperr::SPECK_Storage::view_encoded_bitstream() const -> const vec8_type&
@@ -137,10 +143,14 @@ auto sperr::SPECK_Storage::get_speck_stream_size(const void* buf) const -> uint6
  // go retrieve the value stored in the last 8 bytes of the header
  //
  const uint8_t* const ptr = static_cast<const uint8_t*>(buf);
- uint64_t bit_in_byte;
- std::memcpy(&bit_in_byte, ptr + m_header_size - 8, sizeof(bit_in_byte));
+ uint64_t useful_bits;
+ std::memcpy(&useful_bits, ptr + m_header_size - 8, sizeof(useful_bits));
+
+ auto num_bytes = useful_bits / 8;
+ if (useful_bits % 8 != 0)
+ ++num_bytes;
 
- return (m_header_size + size_t(bit_in_byte));
+ return (m_header_size + num_bytes);
 }
 
 void sperr::SPECK_Storage::set_dimensions(dims_type dims)

utilities/compressor_3d.cpp

@@ -7,6 +7,7 @@
 #include <cstdlib>
 #include <cstring>
 #include <iostream>
+#include <memory>
 
 int main(int argc, char* argv[])
 {
@@ -80,26 +81,26 @@ int main(int argc, char* argv[])
  std::cout << "Compression mode is unclear. Did you give one and only one "
  "compression specification?"
  << std::endl;
- return 1;
+ return __LINE__;
  }
 
  // Do some sanity check on compression parameters
  if (mode == sperr::CompMode::FixedSize) {
  if (bpp <= 0.0 || bpp >= 64.0) {
  std::cout << "Bit-per-pixel value must be between 0.0 and 64.0!" << std::endl;
- return 1;
+ return __LINE__;
  }
  }
  else if (mode == sperr::CompMode::FixedPSNR) {
  if (psnr <= 0.0) {
  std::cout << "Target PSNR must be positive!" << std::endl;
- return 1;
+ return __LINE__;
  }
  }
  else if (mode == sperr::CompMode::FixedPWE) {
  if (pwe <= 0.0) {
  std::cout << "Target PWE must be positive!" << std::endl;
- return 1;
+ return __LINE__;
  }
  }
 
@@ -109,24 +110,24 @@ int main(int argc, char* argv[])
  if ((use_double && orig.size() != total_vals * sizeof(double)) ||
  (!use_double && orig.size() != total_vals * sizeof(float))) {
  std::cerr << "Read input file error: " << input_file << std::endl;
- return 1;
+ return __LINE__;
  }
 
  // Use a compressor and take the input data
- SPERR3D_OMP_C compressor;
- compressor.set_num_threads(omp_num_threads);
+ auto compressor = std::make_unique<SPERR3D_OMP_C>();
+ compressor->set_num_threads(omp_num_threads);
  auto rtn = sperr::RTNType::Good;
  if (use_double) {
- rtn = compressor.copy_data(reinterpret_cast<const double*>(orig.data()), total_vals,
- {dims[0], dims[1], dims[2]}, {chunks[0], chunks[1], chunks[2]});
+ rtn = compressor->copy_data(reinterpret_cast<const double*>(orig.data()), total_vals,
+  {dims[0], dims[1], dims[2]}, {chunks[0], chunks[1], chunks[2]});
  }
  else {
- rtn = compressor.copy_data(reinterpret_cast<const float*>(orig.data()), total_vals,
- {dims[0], dims[1], dims[2]}, {chunks[0], chunks[1], chunks[2]});
+ rtn = compressor->copy_data(reinterpret_cast<const float*>(orig.data()), total_vals,
+  {dims[0], dims[1], dims[2]}, {chunks[0], chunks[1], chunks[2]});
  }
  if (rtn != sperr::RTNType::Good) {
  std::cerr << "Copy data failed!" << std::endl;
- return 1;
+ return __LINE__;
  }
 
  // Free up memory if we don't need to compute stats
@@ -138,63 +139,66 @@ int main(int argc, char* argv[])
  // Tell the compressor which compression mode to use.
  switch (mode) {
  case sperr::CompMode::FixedSize:
- rtn = compressor.set_target_bpp(bpp);
+ rtn = compressor->set_target_bpp(bpp);
  break;
  case sperr::CompMode::FixedPSNR:
- compressor.set_target_psnr(psnr);
+ compressor->set_target_psnr(psnr);
  break;
  default:
- compressor.set_target_pwe(pwe);
+ compressor->set_target_pwe(pwe);
  break;
  }
  if (rtn != sperr::RTNType::Good) {
  std::cerr << "Set bit-per-pixel failed!" << std::endl;
- return 1;
+ return __LINE__;
  }
 
  // Perform the actual compression
- rtn = compressor.compress();
+ rtn = compressor->compress();
  switch (rtn) {
  case sperr::RTNType::QzLevelTooBig:
  std::cerr << "Compression failed because `qz` is set too big!" << std::endl;
- return 1;
+ return __LINE__;
  case sperr::RTNType::Good:
  break;
  default:
  std::cerr << "Compression failed!" << std::endl;
- return 1;
+ return __LINE__;
  }
 
  // Get a hold of the encoded bitstream.
- auto stream = compressor.get_encoded_bitstream();
+ auto stream = compressor->get_encoded_bitstream();
  if (stream.empty()) {
  std::cerr << "Compression bitstream empty!" << std::endl;
- return 1;
+ return __LINE__;
  }
 
  // Write out the encoded bitstream.
  if (!output_file.empty()) {
  rtn = sperr::write_n_bytes(output_file, stream.size(), stream.data());
  if (rtn != sperr::RTNType::Good) {
  std::cerr << "Write compressed file failed!" << std::endl;
- return 1;
+ return __LINE__;
  }
  }
 
  // Calculate and print statistics
  if (show_stats) {
+ const auto out_stats = compressor->get_outlier_stats();
+ compressor.reset(nullptr);
+
  const auto bpp = stream.size() * 8.0 / total_vals;
 
  // Use a decompressor to decompress and collect error statistics
  SPERR3D_OMP_D decompressor;
  decompressor.set_num_threads(omp_num_threads);
  rtn = decompressor.use_bitstream(stream.data(), stream.size());
  if (rtn != RTNType::Good)
- return 1;
+ return __LINE__;
 
  rtn = decompressor.decompress(stream.data());
  if (rtn != RTNType::Good)
- return 1;
+ return __LINE__;
 
  if (use_double) {
  const auto& recover = decompressor.view_data();
@@ -226,8 +230,6 @@ int main(int argc, char* argv[])
  }
 
  if (mode == sperr::CompMode::FixedPWE) {
- // Also collect outlier statistics
- const auto out_stats = compressor.get_outlier_stats();
  if (out_stats.first == 0) {
  std::cout << "There were no outliers corrected!\n";
  }

utilities/show_version.cpp

@@ -4,7 +4,8 @@
 
 int main()
 {
- std::cout << "SPERR version: " << SPERR_VERSION_MAJOR << "." << SPERR_VERSION_MINOR << std::endl;
+ std::cout << "SPERR version: " << SPERR_VERSION_MAJOR << "." << SPERR_VERSION_MINOR << "."
+ << SPERR_VERSION_PATCH << std::endl;
  std::cout << "Based on code Branch: " << SPERR_GIT_BRANCH << std::endl;
  std::cout << "Based on code SHA1 : " << SPERR_GIT_SHA1 << std::endl;
 }