[feat][index] Vector scalar data pre-filter speed increase.

DataCanvasIO · Apr 12, 2024 · 54c2b0d · 54c2b0d
1 parent 7db36b7
commit 54c2b0d
Show file tree

Hide file tree

Showing 32 changed files with 4,633 additions and 113 deletions.
diff --git a/conf/coordinator-gflags.conf b/conf/coordinator-gflags.conf
@@ -17,3 +17,4 @@
 -default_replica_num=3
 -use_pthread_prior_worker_set=true
 -use_prior_worker_set=false
+-dingo_log_switch_scalar_speed_up_detail=true
diff --git a/conf/index-gflags.conf b/conf/index-gflags.conf
@@ -25,3 +25,4 @@
 -dingo_log_switch_txn_detail=true
 -use_pthread_prior_worker_set=true
 -use_prior_worker_set=false
+-dingo_log_switch_scalar_speed_up_detail=true
diff --git a/conf/store-gflags.conf b/conf/store-gflags.conf
@@ -20,3 +20,4 @@
 -dingo_log_switch_txn_detail=true
 -use_pthread_prior_worker_set=true
 -use_prior_worker_set=false
+-dingo_log_switch_scalar_speed_up_detail=true
diff --git a/proto/common.proto b/proto/common.proto
@@ -500,6 +500,7 @@ message VectorIndexParameter {
     CreateDiskAnnParam diskann_parameter = 6;
     CreateBruteForceParam bruteforce_parameter = 7;
   }
+  ScalarSchema scalar_schema = 50;
 }
 
 message SearchFlatParam {
@@ -669,7 +670,6 @@ message ScalarSchema {
 message ScalarIndexParameter {
   ScalarIndexType scalar_index_type = 1;
   bool is_unique = 2;  // for scalar index
-  ScalarSchema scalar_schema = 3;
 }
 
 message IndexParameter {

diff --git a/src/client/coordinator_client_function_meta.cc b/src/client/coordinator_client_function_meta.cc
@@ -57,6 +57,7 @@ DECLARE_int64(tso_new_physical);
 DECLARE_int64(tso_new_logical);
 
 DECLARE_int64(tenant_id);
+DECLARE_bool(with_scalar_schema);
 
 DEFINE_bool(is_updating_index, false, "is index");
 DEFINE_bool(is_index, false, "is index");
@@ -936,6 +937,81 @@ void SendCreateIndex(std::shared_ptr<dingodb::CoordinatorInteraction> coordinato
     index_definition->set_auto_increment(100);
   }
 
+  // scalar key speed up
+  if (FLAGS_with_scalar_schema) {
+    auto* scalar_parameter = vector_index_parameter->mutable_scalar_schema();
+
+    auto* field = scalar_parameter->add_fields();
+    field->set_key("speedup_key_bool");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::BOOL);
+    field->set_enable_speed_up(true);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("speedup_key_int");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::INT32);
+    field->set_enable_speed_up(true);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("speedup_key_long");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::INT64);
+    field->set_enable_speed_up(true);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("speedup_key_float");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::FLOAT32);
+    field->set_enable_speed_up(true);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("speedup_key_double");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::DOUBLE);
+    field->set_enable_speed_up(true);
+
+  field = scalar_parameter->add_fields();
+    field->set_key("speedup_key_string");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::STRING);
+    field->set_enable_speed_up(true);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("speedup_key_bytes");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::BYTES);
+    field->set_enable_speed_up(true);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("key_bool");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::BOOL);
+    field->set_enable_speed_up(false);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("key_int");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::INT32);
+    field->set_enable_speed_up(false);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("key_long");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::INT64);
+    field->set_enable_speed_up(false);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("key_float");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::FLOAT32);
+    field->set_enable_speed_up(false);
+
+  field = scalar_parameter->add_fields();
+    field->set_key("key_double");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::DOUBLE);
+    field->set_enable_speed_up(false);
+
+  field = scalar_parameter->add_fields();
+    field->set_key("key_string");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::STRING);
+    field->set_enable_speed_up(false);
+
+    field = scalar_parameter->add_fields();
+    field->set_key("key_bytes");
+    field->set_field_type(::dingodb::pb::common::ScalarFieldType::BYTES);
+    field->set_enable_speed_up(false);
+  }
+
   DINGO_LOG(INFO) << "Request: " << request.DebugString();
 
   auto status = coordinator_interaction->SendRequest("CreateIndex", request, response);

diff --git a/src/client/dingodb_client.cc b/src/client/dingodb_client.cc
@@ -197,6 +197,9 @@ DEFINE_int64(start_revision, 0, "start revision client supply");
 DEFINE_int64(tenant_id, 0, "tenant id");
 DEFINE_bool(get_all_tenant, false, "get all tenant");
 
+// scalar key speed up
+DEFINE_bool(with_scalar_schema, false, "create vector index with scalar schema");
+
 bvar::LatencyRecorder g_latency_recorder("dingo-store");
 
 const std::map<std::string, std::vector<std::string>> kParamConstraint = {