[feat][index] Implement automatic max_elements expand for hnsw.

Signed-off-by: Ketor <d.ketor@gmail.com>

conf/index-gflags.conf

@@ -3,10 +3,11 @@
 -coor_url=file://./conf/coor_list
 -ip2hostname=false
 -min_system_disk_capacity_free_ratio=0.05
--min_system_memory_capacity_free_ratio=0.10
+-min_system_memory_capacity_free_ratio=0.20
 -omp_num_threads=1
 -max_hnsw_parallel_thread_num=48
 -max_hnsw_parallel_thread_num_per_request=32
 -service_worker_num=16
 -max_short_value_in_write_cf=8
 -vector_max_batch_count=1024
+-hnsw_max_init_max_elements=10000

src/client/coordinator_client_function_meta.cc

@@ -848,8 +848,8 @@ void SendCreateIndex(std::shared_ptr<dingodb::CoordinatorInteraction> coordinato
   }
 
   if (FLAGS_vector_index_type == "hnsw") {
-    if (FLAGS_max_elements == 0) {
-      DINGO_LOG(WARNING) << "max_elements is empty";
+    if (FLAGS_max_elements < 0) {
+      DINGO_LOG(WARNING) << "max_elements is negative";
       return;
     }
     if (FLAGS_efconstruction == 0) {

src/common/helper.cc

@@ -1645,26 +1645,6 @@ butil::Status Helper::ValidateRaftStatusForSplit(std::shared_ptr<pb::common::BRa
   return butil::Status();
 }
 
-// calc hnsw count from memory
-uint32_t Helper::CalcHnswCountFromMemory(int64_t memory_size_limit, int64_t dimension, int64_t nlinks) {
-  // size_links_level0_ = maxM0_ * sizeof(tableint) + sizeof(linklistsizeint);
-  int64_t size_links_level0 = nlinks * 2 + sizeof(int64_t) + sizeof(int64_t);
-
-  // int64_t size_data_per_element_ = size_links_level0_ + data_size_ + sizeof(labeltype);
-  int64_t size_data_per_element = size_links_level0 + sizeof(float_t) * dimension + sizeof(int64_t);
-
-  // int64_t size_link_list_per_element =  sizeof(void*);
-  int64_t size_link_list_per_element = sizeof(int64_t);
-
-  int64_t count = memory_size_limit / (size_data_per_element + size_link_list_per_element);
-
-  if (count > UINT32_MAX) {
-    count = UINT32_MAX;
-  }
-
-  return static_cast<uint32_t>(count);
-}
-
 std::string Helper::GenMaxStartKey() { return std::string(9, '\xff'); }
 
 std::string Helper::GenMinStartKey() { return std::string(1, '\x00'); }

src/common/helper.h

@@ -207,9 +207,6 @@ class Helper {
   static float GenerateRandomFloat(float min_value, float max_value);
   static int64_t GenId();
 
-  // calc hnsw count from memory
-  static uint32_t CalcHnswCountFromMemory(int64_t memory_size_limit, int64_t dimension, int64_t nlinks);
-
   // Gen coordinator new_table_check_name
   static std::string GenNewTableCheckName(int64_t schema_id, const std::string& table_name);
 

src/coordinator/coordinator_control_coor.cc

@@ -42,6 +42,7 @@
 #include "proto/error.pb.h"
 #include "proto/meta.pb.h"
 #include "server/server.h"
+#include "vector/vector_index_hnsw.h"
 #include "vector/vector_index_utils.h"
 
 namespace dingodb {
@@ -67,9 +68,9 @@ DEFINE_int32(
     "region down after secondes, will not update region info if no state change and (now - last_update_time) > "
     "region_down_after_secondes");
 
-DEFINE_int64(max_hnsw_memory_size_of_region, 1024L * 1024L * 1024L, "max memory size of region in HSNW");
+DECLARE_int64(max_hnsw_memory_size_of_region);
 
-DEFINE_int32(max_hnsw_nlinks_of_region, 4096, "max nlinks of region in HSNW");
+DECLARE_int32(max_hnsw_nlinks_of_region);
 
 DEFINE_int32(max_send_region_cmd_per_store, 100, "max send region cmd per store");
 
@@ -1698,18 +1699,11 @@ butil::Status CoordinatorControl::CreateShadowRegion(const std::string& region_n
     // if vector index is hnsw, need to limit max_elements of each region to less than 512MB / dimenstion / 4
     if (new_index_parameter.vector_index_parameter().vector_index_type() ==
         pb::common::VectorIndexType::VECTOR_INDEX_TYPE_HNSW) {
-      auto dimension = new_index_parameter.vector_index_parameter().hnsw_parameter().dimension();
-      auto max_elements = new_index_parameter.vector_index_parameter().hnsw_parameter().max_elements();
-      auto nlinks = new_index_parameter.vector_index_parameter().hnsw_parameter().nlinks();
-
-      auto max_elements_limit =
-          Helper::CalcHnswCountFromMemory(FLAGS_max_hnsw_memory_size_of_region, dimension, nlinks);
-      if (max_elements > max_elements_limit) {
-        DINGO_LOG(WARNING) << "CreateRegion max_elements is too large, will reduce max_elements, max_elements="
-                           << max_elements << ", max_elements_limit=" << max_elements_limit
-                           << ", dimension=" << dimension;
-        new_index_parameter.mutable_vector_index_parameter()->mutable_hnsw_parameter()->set_max_elements(
-            max_elements_limit);
+      auto* hnsw_parameter = new_index_parameter.mutable_vector_index_parameter()->mutable_hnsw_parameter();
+      auto ret1 = VectorIndexHnsw::CheckAndSetHnswParameter(*hnsw_parameter);
+      if (!ret1.ok()) {
+        DINGO_LOG(ERROR) << "CreateRegion vector index region hnsw parameter is not legal, ret=" << ret1;
+        return ret1;
       }
     }
   }
@@ -1860,18 +1854,11 @@ butil::Status CoordinatorControl::CreateRegionFinal(const std::string& region_na
     // if vector index is hnsw, need to limit max_elements of each region to less than 512MB / dimenstion / 4
     if (new_index_parameter.vector_index_parameter().vector_index_type() ==
         pb::common::VectorIndexType::VECTOR_INDEX_TYPE_HNSW) {
-      auto dimension = new_index_parameter.vector_index_parameter().hnsw_parameter().dimension();
-      auto max_elements = new_index_parameter.vector_index_parameter().hnsw_parameter().max_elements();
-      auto nlinks = new_index_parameter.vector_index_parameter().hnsw_parameter().nlinks();
-
-      auto max_elements_limit =
-          Helper::CalcHnswCountFromMemory(FLAGS_max_hnsw_memory_size_of_region, dimension, nlinks);
-      if (max_elements > max_elements_limit) {
-        DINGO_LOG(WARNING) << "CreateRegion max_elements is too large, will reduce max_elements, max_elements="
-                           << max_elements << ", max_elements_limit=" << max_elements_limit
-                           << ", dimension=" << dimension;
-        new_index_parameter.mutable_vector_index_parameter()->mutable_hnsw_parameter()->set_max_elements(
-            max_elements_limit);
+      auto* hnsw_parameter = new_index_parameter.mutable_vector_index_parameter()->mutable_hnsw_parameter();
+      auto ret1 = VectorIndexHnsw::CheckAndSetHnswParameter(*hnsw_parameter);
+      if (!ret1.ok()) {
+        DINGO_LOG(ERROR) << "CreateRegion vector index region hnsw parameter is not legal, ret=" << ret1;
+        return ret1;
       }
     }
   }

src/server/meta_service.cc

@@ -33,6 +33,7 @@
 #include "proto/error.pb.h"
 #include "proto/meta.pb.h"
 #include "server/service_helper.h"
+#include "vector/vector_index_hnsw.h"
 
 namespace dingodb {
 
@@ -1183,28 +1184,14 @@ void DoCreateIndex(google::protobuf::RpcController * /*controller*/, const pb::m
       pb::common::VectorIndexType::VECTOR_INDEX_TYPE_HNSW) {
     auto *hnsw_parameter =
         table_definition.mutable_index_parameter()->mutable_vector_index_parameter()->mutable_hnsw_parameter();
-    if (hnsw_parameter->dimension() <= 0) {
-      DINGO_LOG(ERROR) << "CreateIndex failed in meta_service, hnsw dimension is too small, dimension="
-                       << hnsw_parameter->dimension();
-      response->mutable_error()->set_errcode(pb::error::Errno::EILLEGAL_PARAMTETERS);
-      response->mutable_error()->set_errmsg("hnsw dimension is too small");
+    auto ret1 = VectorIndexHnsw::CheckAndSetHnswParameter(*hnsw_parameter);
+    if (!ret1.ok()) {
+      DINGO_LOG(ERROR) << "CreateIndex failed in meta_service, error code=" << ret1.error_code()
+                       << ", error str=" << ret1.error_str();
+      response->mutable_error()->set_errcode(static_cast<pb::error::Errno>(ret1.error_code()));
+      response->mutable_error()->set_errmsg(ret1.error_str());
       return;
     }
-    auto max_elements_limit = Helper::CalcHnswCountFromMemory(FLAGS_max_hnsw_memory_size_of_region,
-                                                              hnsw_parameter->dimension(), hnsw_parameter->nlinks());
-    if (hnsw_parameter->max_elements() > max_elements_limit) {
-      DINGO_LOG(WARNING) << "CreateIndex warning in meta_service, hnsw max_elements is too big, max_elements="
-                         << hnsw_parameter->max_elements() << ", dimension=" << hnsw_parameter->dimension()
-                         << ", max_memory_size_of_region=" << FLAGS_max_hnsw_memory_size_of_region
-                         << ", max elements in this dimention=" << max_elements_limit;
-      hnsw_parameter->set_max_elements(max_elements_limit);
-    }
-
-    if (hnsw_parameter->nlinks() > FLAGS_max_hnsw_nlinks_of_region) {
-      DINGO_LOG(WARNING) << "CreateIndex warning in meta_service, hnsw nlinks is too big, nlinks="
-                         << hnsw_parameter->nlinks() << ", max_nlinks=" << FLAGS_max_hnsw_nlinks_of_region;
-      hnsw_parameter->set_nlinks(FLAGS_max_hnsw_nlinks_of_region);
-    }
   }
 
   std::vector<int64_t> region_ids;
@@ -1323,28 +1310,13 @@ void DoUpdateIndex(google::protobuf::RpcController * /*controller*/, const pb::m
       pb::common::VectorIndexType::VECTOR_INDEX_TYPE_HNSW) {
     auto *hnsw_parameter =
         table_definition.mutable_index_parameter()->mutable_vector_index_parameter()->mutable_hnsw_parameter();
-    if (hnsw_parameter->dimension() <= 0) {
-      DINGO_LOG(ERROR) << "UpdateIndex failed in meta_service, hnsw dimension is too small, dimension="
-                       << hnsw_parameter->dimension();
-      response->mutable_error()->set_errcode(pb::error::Errno::EILLEGAL_PARAMTETERS);
-      response->mutable_error()->set_errmsg("hnsw dimension is too small");
-    }
-    auto max_elements_limit = Helper::CalcHnswCountFromMemory(FLAGS_max_hnsw_memory_size_of_region,
-                                                              hnsw_parameter->dimension(), hnsw_parameter->nlinks());
-    if (hnsw_parameter->max_elements() > max_elements_limit) {
-      DINGO_LOG(ERROR) << "UpdateIndex warning in meta_service, hnsw max_elements is too big, max_elements="
-                       << hnsw_parameter->max_elements() << ", dimension=" << hnsw_parameter->dimension()
-                       << ", max_memory_size_of_region=" << FLAGS_max_hnsw_memory_size_of_region
-                       << ", max elements in this dimention=" << max_elements_limit;
-      response->mutable_error()->set_errcode(pb::error::Errno::EILLEGAL_PARAMTETERS);
-      response->mutable_error()->set_errmsg("hnsw max_elements is too big");
-    }
-
-    if (hnsw_parameter->nlinks() > FLAGS_max_hnsw_nlinks_of_region) {
-      DINGO_LOG(ERROR) << "UpdateIndex warning in meta_service, hnsw nlinks is too big, nlinks="
-                       << hnsw_parameter->nlinks() << ", max_nlinks=" << FLAGS_max_hnsw_nlinks_of_region;
-      response->mutable_error()->set_errcode(pb::error::Errno::EILLEGAL_PARAMTETERS);
-      response->mutable_error()->set_errmsg("hnsw nlinks is too big");
+    auto ret1 = VectorIndexHnsw::CheckAndSetHnswParameter(*hnsw_parameter);
+    if (!ret1.ok()) {
+      DINGO_LOG(ERROR) << "UpdateIndex failed in meta_service, error code=" << ret1.error_code()
+                       << ", error str=" << ret1.error_str();
+      response->mutable_error()->set_errcode(static_cast<pb::error::Errno>(ret1.error_code()));
+      response->mutable_error()->set_errmsg(ret1.error_str());
+      return;
     }
   }