diff --git a/db/blob/blob_file_builder.cc b/db/blob/blob_file_builder.cc index 70d4dcb93..56d0bbac6 100644 --- a/db/blob/blob_file_builder.cc +++ b/db/blob/blob_file_builder.cc @@ -66,7 +66,7 @@ BlobFileBuilder::BlobFileBuilder( immutable_options_(immutable_options), min_blob_size_(mutable_cf_options->min_blob_size), blob_file_size_(mutable_cf_options->blob_file_size), - blob_compressor_(mutable_cf_options->blob_compressor), + blob_compressor_(mutable_cf_options->derived_blob_compressor), prepopulate_blob_cache_(mutable_cf_options->prepopulate_blob_cache), file_options_(file_options), db_id_(std::move(db_id)), diff --git a/db/column_family.cc b/db/column_family.cc index fdfca69b7..7058adcd1 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -130,17 +130,18 @@ void GetIntTblPropCollectorFactory( Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) { MutableCFOptions moptions(cf_options); ImmutableCFOptions ioptions(cf_options); - if (moptions.compressor && !moptions.compressor->Supported()) { + if (moptions.derived_compressor && + !moptions.derived_compressor->Supported()) { return Status::InvalidArgument("Compression type " + - moptions.compressor->GetId() + + moptions.derived_compressor->GetId() + " is not linked with the binary."); - } else if (moptions.bottommost_compressor && - !moptions.bottommost_compressor->Supported()) { - return Status::InvalidArgument("Compression type " + - moptions.bottommost_compressor->GetId() + - " is not linked with the binary."); - } else if (!moptions.compressor_per_level.empty()) { - for (const auto& compressor : moptions.compressor_per_level) { + } else if (moptions.derived_bottommost_compressor && + !moptions.derived_bottommost_compressor->Supported()) { + return Status::InvalidArgument( + "Compression type " + moptions.derived_bottommost_compressor->GetId() + + " is not linked with the binary."); + } else if (!moptions.derived_compressor_per_level.empty()) { + for (const auto& compressor : moptions.derived_compressor_per_level) { if (compressor == nullptr) { return Status::InvalidArgument("Compression type is invalid."); } else if (!compressor->Supported()) { @@ -170,9 +171,10 @@ Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) { "should be nonzero if we're using zstd's dictionary generator."); } } - if (moptions.blob_compressor && !moptions.blob_compressor->Supported()) { + if (moptions.derived_blob_compressor && + !moptions.derived_blob_compressor->Supported()) { return Status::InvalidArgument("Blob compression type " + - moptions.blob_compressor->GetId() + + moptions.derived_blob_compressor->GetId() + " is not linked with the binary."); } return Status::OK(); diff --git a/db/compaction/compaction_picker.cc b/db/compaction/compaction_picker.cc index fe9f94afc..c3c4d0f78 100644 --- a/db/compaction/compaction_picker.cc +++ b/db/compaction/compaction_picker.cc @@ -87,12 +87,12 @@ std::shared_ptr GetCompressor(const VersionStorageInfo* vstorage, // If bottommost_compression is set and we are compacting to the // bottommost level then we should use it. bool bottom_level = (level >= (vstorage->num_non_empty_levels() - 1)); - if (moptions.bottommost_compressor != nullptr && bottom_level) { - return moptions.bottommost_compressor; + if (moptions.derived_bottommost_compressor != nullptr && bottom_level) { + return moptions.derived_bottommost_compressor; } // If the user has specified a different compression level for each level, // then pick the compression for that level. - if (!moptions.compressor_per_level.empty()) { + if (!moptions.derived_compressor_per_level.empty()) { // It is possible for level_ to be -1; in that case, we use level // 0's compression. This occurs mostly in backwards compatibility // situations when the builder doesn't know what level the file @@ -101,7 +101,8 @@ std::shared_ptr GetCompressor(const VersionStorageInfo* vstorage, assert(level == 0 || level >= base_level); int lvl = std::max(0, level - base_level + 1); int idx = std::min( - static_cast(moptions.compressor_per_level.size()) - 1, lvl); + static_cast(moptions.derived_compressor_per_level.size()) - 1, + lvl); // If not specified directly by the user, compressors in // compressor_per_level are instantiated using compression_opts. If the user // enabled bottommost_compression_opts, we need to create a new compressor @@ -112,10 +113,10 @@ std::shared_ptr GetCompressor(const VersionStorageInfo* vstorage, moptions.compression_per_level[idx], moptions.bottommost_compression_opts); } else { - return moptions.compressor_per_level[idx]; + return moptions.derived_compressor_per_level[idx]; } } else { - return moptions.compressor; + return moptions.derived_compressor; } } diff --git a/db/compaction/compaction_picker_fifo.cc b/db/compaction/compaction_picker_fifo.cc index 32c1d12fb..da22dd4bb 100644 --- a/db/compaction/compaction_picker_fifo.cc +++ b/db/compaction/compaction_picker_fifo.cc @@ -181,7 +181,7 @@ Compaction* FIFOCompactionPicker::PickSizeCompaction( vstorage, ioptions_, mutable_cf_options, mutable_db_options, {comp_inputs}, 0, 16 * 1024 * 1024 /* output file size limit */, 0 /* max compaction bytes, not applicable */, - 0 /* output path ID */, mutable_cf_options.compressor, + 0 /* output path ID */, mutable_cf_options.derived_compressor, Temperature::kUnknown, 0 /* max_subcompactions */, {}, /* is manual */ false, /* trim_ts */ "", vstorage->CompactionScore(0), @@ -416,7 +416,7 @@ Compaction* FIFOCompactionPicker::PickTemperatureChangeCompaction( vstorage, ioptions_, mutable_cf_options, mutable_db_options, std::move(inputs), 0, 0 /* output file size limit */, 0 /* max compaction bytes, not applicable */, 0 /* output path ID */, - mutable_cf_options.compressor, compaction_target_temp, + mutable_cf_options.derived_compressor, compaction_target_temp, /* max_subcompactions */ 0, {}, /* is manual */ false, /* trim_ts */ "", vstorage->CompactionScore(0), /* is deletion compaction */ false, /* l0_files_might_overlap */ true, diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index 10cd313ec..faab7887c 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -138,14 +138,14 @@ std::shared_ptr GetCompressionFlush( // latency overhead is not offset by saving much space. if (ioptions.compaction_style == kCompactionStyleUniversal) { if (moptions.compaction_options_universal.compression_size_percent < 0) { - return moptions.compressor; + return moptions.derived_compressor; } else { return BuiltinCompressor::GetCompressor(kNoCompression); } - } else if (moptions.compressor_per_level.empty()) { - return moptions.compressor; + } else if (moptions.derived_compressor_per_level.empty()) { + return moptions.derived_compressor; } else { - return moptions.compressor_per_level[0]; + return moptions.derived_compressor_per_level[0]; } } diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index 295d232e4..86431fa66 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -1931,8 +1931,9 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) { ->compaction_style) /* output file size limit, not applicable */ , LLONG_MAX /* max compaction bytes, not applicable */, - 0 /* output path ID, not applicable */, mutable_cf_options.compressor, - Temperature::kUnknown, 0 /* max_subcompactions, not applicable */, + 0 /* output path ID, not applicable */, + mutable_cf_options.derived_compressor, Temperature::kUnknown, + 0 /* max_subcompactions, not applicable */, {} /* grandparents, not applicable */, false /* is manual */, "" /* trim_ts */, -1 /* score, not applicable */, false /* is deletion compaction, not applicable */, @@ -4155,7 +4156,7 @@ void DBImpl::BuildCompactionJobInfo( newf.first, file_number, meta.oldest_blob_file_number}); } compaction_job_info->blob_compression_type = - c->mutable_cf_options()->blob_compressor->GetCompressionType(); + c->mutable_cf_options()->derived_blob_compressor->GetCompressionType(); // Update BlobFilesInfo. for (const auto& blob_file : c->edit()->GetBlobFileAdditions()) { diff --git a/db/external_sst_file_ingestion_job.cc b/db/external_sst_file_ingestion_job.cc index bbd02dd7d..dfe0bb1c2 100644 --- a/db/external_sst_file_ingestion_job.cc +++ b/db/external_sst_file_ingestion_job.cc @@ -550,8 +550,9 @@ void ExternalSstFileIngestionJob::CreateEquivalentFileIngestingCompactions() { */ , LLONG_MAX /* max compaction bytes, not applicable */, - 0 /* output path ID, not applicable */, mutable_cf_options.compressor, - Temperature::kUnknown, 0 /* max_subcompaction, not applicable */, + 0 /* output path ID, not applicable */, + mutable_cf_options.derived_compressor, Temperature::kUnknown, + 0 /* max_subcompaction, not applicable */, {} /* grandparents, not applicable */, false /* is manual */, "" /* trim_ts */, -1 /* score, not applicable */, false /* is deletion compaction, not applicable */, diff --git a/db/flush_job.cc b/db/flush_job.cc index 2e5328d56..2c5126ffa 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -1134,7 +1134,7 @@ std::unique_ptr FlushJob::GetFlushJobInfo() const { info->table_properties = table_properties_; info->flush_reason = flush_reason_; info->blob_compression_type = - mutable_cf_options_.blob_compressor->GetCompressionType(); + mutable_cf_options_.derived_blob_compressor->GetCompressionType(); // Update BlobFilesInfo. for (const auto& blob_file : edit_->GetBlobFileAdditions()) { diff --git a/db/table_properties_collector_test.cc b/db/table_properties_collector_test.cc index a230d0ab2..9009d3148 100644 --- a/db/table_properties_collector_test.cc +++ b/db/table_properties_collector_test.cc @@ -51,10 +51,10 @@ void MakeBuilder( std::unique_ptr wf(new test::StringSink); writable->reset( new WritableFileWriter(std::move(wf), "" /* don't care */, EnvOptions())); - TableBuilderOptions tboptions(ioptions, moptions, internal_comparator, - int_tbl_prop_collector_factories, - moptions.compressor, kTestColumnFamilyId, - kTestColumnFamilyName, kTestLevel); + TableBuilderOptions tboptions( + ioptions, moptions, internal_comparator, int_tbl_prop_collector_factories, + moptions.derived_compressor, kTestColumnFamilyId, kTestColumnFamilyName, + kTestLevel); builder->reset(NewTableBuilder(tboptions, writable->get())); } } // namespace diff --git a/options/cf_options.cc b/options/cf_options.cc index ff856c13d..9bb5a0ee9 100644 --- a/options/cf_options.cc +++ b/options/cf_options.cc @@ -1211,41 +1211,54 @@ void MutableCFOptions::RefreshDerivedOptions(int num_levels, } } + derived_compressor.reset(); if (compressor == nullptr) { - compressor = + derived_compressor = BuiltinCompressor::GetCompressor(compression, compression_opts); - } - if (compressor == nullptr) { - compressor = BuiltinCompressor::GetCompressor(kSnappyCompression); + if (derived_compressor == nullptr) { + derived_compressor = BuiltinCompressor::GetCompressor(kSnappyCompression); + } + } else { + derived_compressor = compressor; } + derived_bottommost_compressor.reset(); if (bottommost_compressor == nullptr) { if (bottommost_compression != kDisableCompressionOption) { if (bottommost_compression_opts.enabled) { - bottommost_compressor = BuiltinCompressor::GetCompressor( + derived_bottommost_compressor = BuiltinCompressor::GetCompressor( bottommost_compression, bottommost_compression_opts); } else { - bottommost_compressor = BuiltinCompressor::GetCompressor( + derived_bottommost_compressor = BuiltinCompressor::GetCompressor( bottommost_compression, compression_opts); } } + } else { + derived_bottommost_compressor = bottommost_compressor; } + derived_blob_compressor.reset(); if (blob_compressor == nullptr) { if (blob_compression_type != kDisableCompressionOption) { - blob_compressor = BuiltinCompressor::GetCompressor(blob_compression_type, - compression_opts); + derived_blob_compressor = BuiltinCompressor::GetCompressor( + blob_compression_type, compression_opts); } - } - if (blob_compressor == nullptr) { - blob_compressor = BuiltinCompressor::GetCompressor(kNoCompression); + if (derived_blob_compressor == nullptr) { + derived_blob_compressor = + BuiltinCompressor::GetCompressor(kNoCompression); + } + } else { + derived_blob_compressor = blob_compressor; } + derived_compressor_per_level.clear(); if (compressor_per_level.empty()) { for (auto type : compression_per_level) { - compressor_per_level.push_back( + derived_compressor_per_level.push_back( BuiltinCompressor::GetCompressor(type, compression_opts)); } + } else { + derived_compressor_per_level = compressor_per_level; } } diff --git a/options/cf_options.h b/options/cf_options.h index ddce608a7..5ee9f302f 100644 --- a/options/cf_options.h +++ b/options/cf_options.h @@ -230,6 +230,10 @@ struct MutableCFOptions { // Derived options // Per-level target file size. std::vector max_file_size; + std::shared_ptr derived_compressor; + std::shared_ptr derived_bottommost_compressor; + std::shared_ptr derived_blob_compressor; + std::vector> derived_compressor_per_level; }; uint64_t MultiplyCheckOverflow(uint64_t op1, double op2); diff --git a/options/options.cc b/options/options.cc index 42cc0d241..df279f344 100644 --- a/options/options.cc +++ b/options/options.cc @@ -188,14 +188,17 @@ void ColumnFamilyOptions::Dump(Logger* log) const { MutableCFOptions moptions(*this); ConfigOptions config_options; - if (!moptions.compressor_per_level.empty()) { - for (unsigned int i = 0; i < moptions.compressor_per_level.size(); i++) { - ROCKS_LOG_HEADER(log, " Options.compression[%d]: %s", i, - moptions.compressor_per_level[i]->GetId().c_str()); + if (!moptions.derived_compressor_per_level.empty()) { + for (unsigned int i = 0; i < moptions.derived_compressor_per_level.size(); + i++) { + ROCKS_LOG_HEADER( + log, " Options.compression[%d]: %s", i, + moptions.derived_compressor_per_level[i]->GetId().c_str()); } - } else if (moptions.compressor) { - ROCKS_LOG_HEADER(log, " Options.compression: %s", - moptions.compressor->ToString(config_options).c_str()); + } else if (moptions.derived_compressor) { + ROCKS_LOG_HEADER( + log, " Options.compression: %s", + moptions.derived_compressor->ToString(config_options).c_str()); } else { ROCKS_LOG_HEADER( log, " Options.compression: %s", @@ -228,10 +231,11 @@ void ColumnFamilyOptions::Dump(Logger* log) const { "%" PRIu64, compression_opts.max_dict_buffer_bytes); } - if (moptions.bottommost_compressor) { + if (moptions.derived_bottommost_compressor) { ROCKS_LOG_HEADER( log, " Options.bottommost_compression: %s", - moptions.bottommost_compressor->ToString(config_options).c_str()); + moptions.derived_bottommost_compressor->ToString(config_options) + .c_str()); } else { ROCKS_LOG_HEADER( log, " Options.bottommost_compression: %s", @@ -451,9 +455,9 @@ void ColumnFamilyOptions::Dump(Logger* log) const { ROCKS_LOG_HEADER(log, " Options.blob_file_size: %" PRIu64, blob_file_size); - if (moptions.blob_compressor) { + if (moptions.derived_blob_compressor) { ROCKS_LOG_HEADER(log, " Options.blob_compression: %s", - moptions.blob_compressor->GetId().c_str()); + moptions.derived_blob_compressor->GetId().c_str()); } else { ROCKS_LOG_HEADER( log, " Options.blob_compression_type: %s", diff --git a/options/options_settable_test.cc b/options/options_settable_test.cc index 0a5450e0e..2985b0d23 100644 --- a/options/options_settable_test.cc +++ b/options/options_settable_test.cc @@ -646,6 +646,8 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) { sizeof(std::vector>)}, {offsetof(struct MutableCFOptions, max_file_size), sizeof(std::vector)}, + {offsetof(struct MutableCFOptions, derived_compressor_per_level), + sizeof(std::vector>)}, }; // For all memory used for options, pre-fill every char. Otherwise, the diff --git a/table/block_based/data_block_hash_index_test.cc b/table/block_based/data_block_hash_index_test.cc index 3fb9aab8c..d162dcc85 100644 --- a/table/block_based/data_block_hash_index_test.cc +++ b/table/block_based/data_block_hash_index_test.cc @@ -556,7 +556,7 @@ void TestBoundary(InternalKey& ik1, std::string& v1, InternalKey& ik2, builder.reset(ioptions.table_factory->NewTableBuilder( TableBuilderOptions( ioptions, moptions, internal_comparator, - &int_tbl_prop_collector_factories, moptions.compressor, + &int_tbl_prop_collector_factories, moptions.derived_compressor, TablePropertiesCollectorFactory::Context::kUnknownColumnFamily, column_family_name, level_), file_writer.get())); diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc index 56c94a675..9dd08274f 100644 --- a/table/sst_file_writer.cc +++ b/table/sst_file_writer.cc @@ -311,14 +311,14 @@ Status SstFileWriter::Open(const std::string& file_path) { sst_file->SetIOPriority(r->io_priority); std::shared_ptr compressor; - if (r->mutable_cf_options.bottommost_compressor != nullptr) { - compressor = r->mutable_cf_options.bottommost_compressor; - } else if (r->mutable_cf_options.compressor_per_level.empty()) { - compressor = r->mutable_cf_options.compressor; + if (r->mutable_cf_options.derived_bottommost_compressor != nullptr) { + compressor = r->mutable_cf_options.derived_bottommost_compressor; + } else if (r->mutable_cf_options.derived_compressor_per_level.empty()) { + compressor = r->mutable_cf_options.derived_compressor; } else { // Use the compression of the last level if we have per level compression - auto levels = r->mutable_cf_options.compressor_per_level.size(); - compressor = r->mutable_cf_options.compressor_per_level[levels - 1]; + auto levels = r->mutable_cf_options.derived_compressor_per_level.size(); + compressor = r->mutable_cf_options.derived_compressor_per_level[levels - 1]; } IntTblPropCollectorFactories int_tbl_prop_collector_factories; diff --git a/table/table_test.cc b/table/table_test.cc index 05205d32f..aa4c437aa 100644 --- a/table/table_test.cc +++ b/table/table_test.cc @@ -400,7 +400,7 @@ class TableConstructor : public Constructor { builder.reset(ioptions.table_factory->NewTableBuilder( TableBuilderOptions(ioptions, moptions, internal_comparator, &int_tbl_prop_collector_factories, - moptions.compressor, kUnknownColumnFamily, + moptions.derived_compressor, kUnknownColumnFamily, column_family_name, level_), file_writer_.get())); @@ -3995,9 +3995,10 @@ TEST_P(BlockBasedTableTest, NoFileChecksum) { f.CreateWritableFile(); std::unique_ptr builder; builder.reset(ioptions.table_factory->NewTableBuilder( - TableBuilderOptions( - ioptions, moptions, *comparator, &int_tbl_prop_collector_factories, - moptions.compressor, kUnknownColumnFamily, column_family_name, level), + TableBuilderOptions(ioptions, moptions, *comparator, + &int_tbl_prop_collector_factories, + moptions.derived_compressor, kUnknownColumnFamily, + column_family_name, level), f.GetFileWriter())); ASSERT_OK(f.ResetTableBuilder(std::move(builder))); f.AddKVtoKVMap(1000); @@ -4030,9 +4031,10 @@ TEST_P(BlockBasedTableTest, Crc32cFileChecksum) { f.SetFileChecksumGenerator(checksum_crc32c_gen1.release()); std::unique_ptr builder; builder.reset(ioptions.table_factory->NewTableBuilder( - TableBuilderOptions( - ioptions, moptions, *comparator, &int_tbl_prop_collector_factories, - moptions.compressor, kUnknownColumnFamily, column_family_name, level), + TableBuilderOptions(ioptions, moptions, *comparator, + &int_tbl_prop_collector_factories, + moptions.derived_compressor, kUnknownColumnFamily, + column_family_name, level), f.GetFileWriter())); ASSERT_OK(f.ResetTableBuilder(std::move(builder))); f.AddKVtoKVMap(1000); diff --git a/util/compression_test.cc b/util/compression_test.cc index 9393c6782..80d40cd20 100644 --- a/util/compression_test.cc +++ b/util/compression_test.cc @@ -494,8 +494,7 @@ TEST(Compression, ColumnFamilyOptionsFromStringWithCompression) { config_options, options, "compression=kZlibCompression", &new_options); ASSERT_OK(s); ASSERT_EQ(new_options.compression, kZlibCompression); - ASSERT_NE(new_options.compressor, nullptr); - ASSERT_STREQ(new_options.compressor->Name(), ZlibCompressor::kClassName()); + ASSERT_EQ(new_options.compressor, nullptr); } TEST(Compression, StringFromColumnFamilyOptions) { @@ -536,8 +535,7 @@ TEST(Compression, StringFromColumnFamilyOptions) { std::string::npos); ASSERT_TRUE(opts_serialized.find("bottommost_compressor=nullptr") != std::string::npos); - ASSERT_TRUE(opts_serialized.find("blob_compressor={id=" + - std::string(NoCompressor::kClassName())) != + ASSERT_TRUE(opts_serialized.find("blob_compressor=nullptr") != std::string::npos); // Re-parse serialized options @@ -745,10 +743,7 @@ TEST(Compression, DBWithZlibAndCompressionOptions) { ASSERT_OK(s); ASSERT_EQ(cf_descs[0].options.compression, kZlibCompression); ASSERT_EQ(cf_descs[0].options.compression_opts.window_bits, -13); - ASSERT_STREQ(cf_descs[0].options.compressor->ToString(config_options).c_str(), - "id=Zlib;max_dict_bytes=0;max_train_bytes=0;max_dict_buffer_" - "bytes=0;strategy=0;window_bits=-13;use_zstd_dict_trainer=true;" - "level=32767;parallel_threads=1"); + ASSERT_EQ(cf_descs[0].options.compressor, nullptr); CloseDB(db); ASSERT_OK(DestroyDB(dbname, options)); @@ -782,17 +777,7 @@ TEST(Compression, DBWithCompressionPerLevel) { ASSERT_EQ(cf_descs[0].options.compression_per_level.size(), 2); ASSERT_EQ(cf_descs[0].options.compression_per_level[0], kNoCompression); ASSERT_EQ(cf_descs[0].options.compression_per_level[1], kSnappyCompression); - ASSERT_EQ(cf_descs[0].options.compressor_per_level.size(), 2); - ASSERT_STREQ(cf_descs[0] - .options.compressor_per_level[0] - ->ToString(config_options) - .c_str(), - "id=NoCompression;parallel_threads=1"); - ASSERT_STREQ(cf_descs[0] - .options.compressor_per_level[1] - ->ToString(config_options) - .c_str(), - "id=Snappy;parallel_threads=1"); + ASSERT_EQ(cf_descs[0].options.compressor_per_level.size(), 0); CloseDB(db);