[feat][store] Support integration test.

CMakeLists.txt

@@ -37,6 +37,7 @@ project(dingo-store C CXX)
 option(EXAMPLE_LINK_SO "Whether examples are linked dynamically" OFF)
 option(LINK_TCMALLOC "Link tcmalloc if possible" OFF)
 option(BUILD_UNIT_TESTS "Build unit test" ON)
+option(BUILD_INTEGRATION_TESTS "Build integration test" OFF)
 option(DINGO_BUILD_STATIC "Link libraries statically to generate the DingoDB binary" ON)
 option(ENABLE_FAILPOINT "Enable failpoint" OFF)
 option(WITH_DISKANN "Build with diskann index" OFF)
@@ -478,5 +479,9 @@ add_subdirectory(src/sdk)
 add_subdirectory(src/example)
 
 if(BUILD_UNIT_TESTS)
-    add_subdirectory(test)
+    add_subdirectory(test/unit_test)
+endif()
+
+if(BUILD_INTEGRATION_TESTS)
+    add_subdirectory(test/integration_test)
 endif()

src/sdk/coordinator_proxy.cc

@@ -65,6 +65,17 @@ Status CoordinatorProxy::CreateRegion(const pb::coordinator::CreateRegionRequest
   return Status::OK();
 }
 
+Status CoordinatorProxy::DropRegion(const pb::coordinator::DropRegionRequest& request,
+                                    pb::coordinator::DropRegionResponse& response) {
+  butil::Status rpc_status = coordinator_interaction_->SendRequest("DropRegion", request, response);
+  if (!rpc_status.ok()) {
+    std::string msg =
+        fmt::format("DropRegion fail, code: {}, msg:{}", rpc_status.error_code(), rpc_status.error_cstr());
+    return Status::NetworkError(msg);
+  }
+  return Status::OK();
+}
+
 Status CoordinatorProxy::ScanRegions(const pb::coordinator::ScanRegionsRequest& request,
                                      pb::coordinator::ScanRegionsResponse& response) {
   butil::Status rpc_status = coordinator_interaction_->SendRequest("ScanRegions", request, response);

src/sdk/coordinator_proxy.h

@@ -23,6 +23,7 @@
 
 namespace dingodb {
 namespace sdk {
+
 class CoordinatorProxy {
  public:
   CoordinatorProxy(const CoordinatorProxy&) = delete;
@@ -38,6 +39,9 @@ class CoordinatorProxy {
   virtual Status CreateRegion(const pb::coordinator::CreateRegionRequest& request,
                               pb::coordinator::CreateRegionResponse& response);
 
+  virtual Status DropRegion(const pb::coordinator::DropRegionRequest& request,
+                            pb::coordinator::DropRegionResponse& response);
+
   virtual Status ScanRegions(const pb::coordinator::ScanRegionsRequest& request,
                              pb::coordinator::ScanRegionsResponse& response);
 
@@ -49,6 +53,7 @@ class CoordinatorProxy {
   std::shared_ptr<dingodb::CoordinatorInteraction> coordinator_interaction_meta_;
   std::shared_ptr<dingodb::CoordinatorInteraction> coordinator_interaction_version_;
 };
+
 }  // namespace sdk
 
 }  // namespace dingodb

src/server/coordinator_service.cc

@@ -1056,9 +1056,9 @@ void DoCreateRegion(google::protobuf::RpcController * /*controller*/,
     DINGO_LOG(ERROR) << "Create Region Failed, errno=" << ret << " Request:" << request->DebugString();
     response->mutable_error()->set_errcode(static_cast<pb::error::Errno>(ret.error_code()));
     response->mutable_error()->set_errmsg(ret.error_str());
-    response->set_region_id(new_region_id);
     return;
   }
+  response->set_region_id(new_region_id);
 
   // if meta_increment is empty, means no need to update meta
   if (meta_increment.ByteSizeLong() == 0) {

test/integration_test/CMakeLists.txt

@@ -0,0 +1,24 @@
+enable_testing()
+
+SET(INTEGRATION_TEST_BIN "dingodb_integration_test")
+
+file(GLOB INTEGRATION_TEST_SRCS "test_*.cc")
+
+add_executable(${INTEGRATION_TEST_BIN}
+                main.cc
+                ${INTEGRATION_TEST_SRCS}
+                )
+
+add_dependencies(${INTEGRATION_TEST_BIN} sdk fmt gtest glog)
+
+target_link_libraries(${INTEGRATION_TEST_BIN}
+                      PRIVATE
+                      $<TARGET_OBJECTS:PROTO_OBJS>
+                      sdk
+                      ${PROTOBUF_LIBRARY}
+                      ${GFLAGS_LIBRARIES}
+                      ${FMT_LIBRARIES}
+                      ${GLOG_LIBRARIES}
+                      ${GTEST_LIBRARIES}
+                      ${BRPC_LIBRARIES}
+                      )

test/integration_test/environment.h

@@ -0,0 +1,61 @@
+// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DINGODB_INTEGRATION_TEST_ENVIROMENT_
+#define DINGODB_INTEGRATION_TEST_ENVIROMENT_
+
+#include <iostream>
+
+#include "fmt/core.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "sdk/client.h"
+#include "sdk/coordinator_proxy.h"
+
+DECLARE_string(coordinator_url);
+
+namespace dingodb {
+
+namespace integration_test {
+
+class Environment : public testing::Environment {
+ public:
+  Environment() : coordinator_proxy_(std::make_shared<sdk::CoordinatorProxy>()) {}
+  static Environment& GetInstance() {
+    static Environment environment;
+    return environment;
+  }
+
+  void SetUp() override {
+    auto status = coordinator_proxy_->Open(FLAGS_coordinator_url);
+    CHECK(status.IsOK()) << "Open coordinator proxy failed, please check parameter --url=" << FLAGS_coordinator_url;
+
+    status = sdk::Client::Build(FLAGS_coordinator_url, client_);
+    CHECK(status.IsOK()) << fmt::format("Build sdk client failed, error: {}", status.ToString());
+  }
+  void TearDown() override {}
+
+  std::shared_ptr<sdk::CoordinatorProxy> GetCoordinatorProxy() { return coordinator_proxy_; }
+  std::shared_ptr<sdk::Client> GetClient() { return client_; }
+
+ private:
+  std::shared_ptr<sdk::CoordinatorProxy> coordinator_proxy_;
+  std::shared_ptr<sdk::Client> client_;
+};
+
+}  // namespace integration_test
+
+}  // namespace dingodb
+
+#endif  // DINGODB_INTEGRATION_TEST_ENVIROMENT_

test/integration_test/helper.h

@@ -0,0 +1,106 @@
+// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef DINGODB_INTEGRATION_TEST_HELPER_
+#define DINGODB_INTEGRATION_TEST_HELPER_
+
+#include <cstdint>
+#include <iostream>
+#include <string>
+
+#include "environment.h"
+#include "fmt/core.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+#include "proto/coordinator.pb.h"
+
+namespace dingodb {
+
+namespace integration_test {
+
+static const std::string kClientRaw = "w";
+
+class Helper {
+ public:
+  static std::string EncodeRawKey(const std::string& str) { return kClientRaw + str; }
+
+  static std::string PrefixNext(const std::string& input) {
+    std::string ret(input.size(), 0);
+    int carry = 1;
+    for (int i = input.size() - 1; i >= 0; --i) {
+      if (static_cast<uint8_t>(input[i]) == (uint8_t)0xFF && carry == 1) {
+        ret[i] = 0;
+      } else {
+        ret[i] = (input[i] + carry);
+        carry = 0;
+      }
+    }
+
+    return (carry == 0) ? ret : input;
+  }
+
+  static int64_t CreateRawRegion(const std::string& name, const std::string& start_key, const std::string& end_key,
+                                 int replicas = 3) {
+    CHECK(!name.empty()) << "name should not empty";
+    CHECK(!start_key.empty()) << "start_key should not empty";
+    CHECK(!end_key.empty()) << "end_key should not empty";
+    CHECK(start_key < end_key) << "start_key must < end_key";
+    CHECK(replicas > 0) << "replicas must > 0";
+
+    pb::coordinator::CreateRegionRequest request;
+    pb::coordinator::CreateRegionResponse response;
+
+    request.set_region_name(name);
+    request.set_replica_num(replicas);
+    request.mutable_range()->set_start_key(EncodeRawKey(start_key));
+    request.mutable_range()->set_end_key(EncodeRawKey(end_key));
+
+    LOG(INFO) << "Create region request: " << request.ShortDebugString();
+
+    auto status = Environment::GetInstance().GetCoordinatorProxy()->CreateRegion(request, response);
+    CHECK(status.IsOK()) << fmt::format("Create region failed, {}", status.ToString());
+    CHECK(response.region_id() != 0) << "region_id is invalid";
+    return response.region_id();
+  }
+
+  static void DropRawRegion(int64_t region_id) {
+    CHECK(region_id != 0) << "region_id is invalid";
+
+    pb::coordinator::DropRegionRequest request;
+    pb::coordinator::DropRegionResponse response;
+
+    request.set_region_id(region_id);
+
+    LOG(INFO) << "Drop region request: " << request.ShortDebugString();
+
+    auto status = Environment::GetInstance().GetCoordinatorProxy()->DropRegion(request, response);
+    CHECK(status.IsOK()) << fmt::format("Drop region failed, {}", status.ToString());
+  }
+
+  static bool IsContain(const std::vector<sdk::KVPair>& kvs, const std::string& key) {
+    for (auto& kv : kvs) {
+      if (kv.key == key) {
+        return true;
+      }
+    }
+
+    return false;
+  }
+};
+
+}  // namespace integration_test
+
+}  // namespace dingodb
+
+#endif  // DINGODB_INTEGRATION_TEST_HELPER_

test/integration_test/main.cc

@@ -0,0 +1,53 @@
+// Copyright (c) 2023 dingodb.com, Inc. All Rights Reserved
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+
+#include "environment.h"
+#include "fmt/core.h"
+#include "glog/logging.h"
+#include "gtest/gtest.h"
+
+DEFINE_string(coordinator_url, "file://./coor_list", "coordinator url");
+DEFINE_int32(create_region_wait_time_s, 3, "create region wait time");
+
+int main(int argc, char* argv[]) {
+  testing::AddGlobalTestEnvironment(&dingodb::integration_test::Environment::GetInstance());
+  testing::InitGoogleTest(&argc, argv);
+  google::ParseCommandLineFlags(&argc, &argv, true);
+
+  int ret = RUN_ALL_TESTS();
+
+  // LOG(INFO) << fmt::format(
+  //     "Test summary: test_suite(total({})/success({})/fail({})) test_case(total({})/success({})/fail({})/skip({})) ",
+  //     testing::UnitTest::GetInstance()->total_test_suite_count(),
+  //     testing::UnitTest::GetInstance()->successful_test_suite_count(),
+  //     testing::UnitTest::GetInstance()->failed_test_suite_count(),
+  //     testing::UnitTest::GetInstance()->total_test_count(),
+  //     testing::UnitTest::GetInstance()->successful_test_count(),
+  //     testing::UnitTest::GetInstance()->failed_test_count(), testing::UnitTest::GetInstance()->skipped_test_count());
+
+  // int total_count = testing::UnitTest::GetInstance()->total_test_suite_count();
+  // for (int i = 0; i < total_count; ++i) {
+  //   const auto* test_suite = testing::UnitTest::GetInstance()->GetTestSuite(i);
+  //   int total_case_count = test_suite->total_test_count();
+  //   for (int j = 0; j < total_case_count; ++j) {
+  //     const auto* test_info = test_suite->GetTestInfo(j);
+  //     test_info->result();
+  //   }
+  // }
+  // const auto& test_result = testing::UnitTest::GetInstance()->ad_hoc_test_result();
+
+  return ret;
+}