Initial implementation of the rpcdb grpc-service (#230)

.github/workflows/ci.yaml

@@ -19,6 +19,7 @@ jobs:
         with:
           toolchain: stable
           override: true
+      - uses: arduino/setup-protoc@v2
       - name: Restore Cargo Cache
         id: cargo-cache
         uses: actions/cache/restore@v3
@@ -83,6 +84,7 @@ jobs:
           toolchain: stable
           override: true
           components: rustfmt, clippy
+      - uses: arduino/setup-protoc@v2
       - name: Restore Check Deps
         id: cache-build-deps-restore
         uses: actions/cache/restore@v3
@@ -109,6 +111,7 @@ jobs:
         with:
           toolchain: stable
           override: true
+      - uses: arduino/setup-protoc@v2
       - name: Restore Check Deps
         id: cache-build-deps-restore
         uses: actions/cache/restore@v3
@@ -132,6 +135,7 @@ jobs:
         with:
           toolchain: stable
           override: true
+      - uses: arduino/setup-protoc@v2
       - name: Restore Check Deps
         id: cache-build-deps-restore
         uses: actions/cache/restore@v3
@@ -158,6 +162,7 @@ jobs:
         with:
           toolchain: stable
           override: true
+      - uses: arduino/setup-protoc@v2
       - name: Restore Check Deps
         id: cache-build-deps-restore
         uses: actions/cache/restore@v3

Cargo.toml

@@ -1,10 +1,11 @@
 [workspace]
 members = [
+  "firewood",
+  "fwdctl",
   "growth-ring",
   "libaio",
+  "rpc",
   "shale",
-  "firewood",
-  "fwdctl",
 ]
 resolver = "2"
 

rpc/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "rpc"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[[bin]]
+name = "server"
+test = false
+bench = false
+
+[[bin]]
+name = "client"
+test = false
+bench = false
+
+[dependencies]
+firewood = { version = "0.0.3", path = "../firewood" }
+prost = "0.11.9"
+thiserror = "1.0.47"
+tokio = { version = "1.32.0", features = ["sync", "rt-multi-thread"] }
+tonic = { version = "0.9.2", features = ["tls"] }
+
+[build-dependencies]
+tonic-build = "0.9.2"

rpc/build.rs

@@ -0,0 +1,6 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    tonic_build::compile_protos("proto/sync/sync.proto")?;
+    tonic_build::compile_protos("proto/rpcdb/rpcdb.proto")?;
+
+    Ok(())
+}

rpc/proto/rpcdb/rpcdb.proto

@@ -0,0 +1,127 @@
+syntax = "proto3";
+
+package rpcdb;
+
+import "google/protobuf/empty.proto";
+
+option go_package = "github.com/ava-labs/avalanchego/proto/pb/rpcdb";
+
+service Database {
+  rpc Has(HasRequest) returns (HasResponse);
+  rpc Get(GetRequest) returns (GetResponse);
+  rpc Put(PutRequest) returns (PutResponse);
+  rpc Delete(DeleteRequest) returns (DeleteResponse);
+  rpc Compact(CompactRequest) returns (CompactResponse);
+  rpc Close(CloseRequest) returns (CloseResponse);
+  rpc HealthCheck(google.protobuf.Empty) returns (HealthCheckResponse);
+  rpc WriteBatch(WriteBatchRequest) returns (WriteBatchResponse);
+  rpc NewIteratorWithStartAndPrefix(NewIteratorWithStartAndPrefixRequest) returns (NewIteratorWithStartAndPrefixResponse);
+  rpc IteratorNext(IteratorNextRequest) returns (IteratorNextResponse);
+  rpc IteratorError(IteratorErrorRequest) returns (IteratorErrorResponse);
+  rpc IteratorRelease(IteratorReleaseRequest) returns (IteratorReleaseResponse);
+}
+
+enum Error {
+  // ERROR_UNSPECIFIED is used to indicate that no error occurred.
+  ERROR_UNSPECIFIED = 0;
+  ERROR_CLOSED = 1;
+  ERROR_NOT_FOUND = 2;
+}
+
+message HasRequest {
+  bytes key = 1;
+}
+
+message HasResponse {
+  bool has = 1;
+  Error err = 2;
+}
+
+message GetRequest {
+  bytes key = 1;
+}
+
+message GetResponse {
+  bytes value = 1;
+  Error err = 2;
+}
+
+message PutRequest {
+  bytes key = 1;
+  bytes value = 2;
+}
+
+message PutResponse {
+  Error err = 1;
+}
+
+message DeleteRequest {
+  bytes key = 1;
+}
+
+message DeleteResponse {
+  Error err = 1;
+}
+
+message CompactRequest {
+  bytes start = 1;
+  bytes limit = 2;
+}
+
+message CompactResponse {
+  Error err = 1;
+}
+
+message CloseRequest {}
+
+message CloseResponse {
+  Error err = 1;
+}
+
+message WriteBatchRequest {
+  repeated PutRequest puts = 1;
+  repeated DeleteRequest deletes = 2;
+}
+
+message WriteBatchResponse {
+  Error err = 1;
+}
+
+message NewIteratorRequest {}
+
+message NewIteratorWithStartAndPrefixRequest {
+  bytes start = 1;
+  bytes prefix = 2;
+}
+
+message NewIteratorWithStartAndPrefixResponse {
+  uint64 id = 1;
+}
+
+message IteratorNextRequest {
+  uint64 id = 1;
+}
+
+message IteratorNextResponse {
+  repeated PutRequest data = 1;
+}
+
+message IteratorErrorRequest {
+  uint64 id = 1;
+}
+
+message IteratorErrorResponse {
+  Error err = 1;
+}
+
+message IteratorReleaseRequest {
+  uint64 id = 1;
+}
+
+message IteratorReleaseResponse {
+  Error err = 1;
+}
+
+message HealthCheckResponse {
+  bytes details = 1;
+}

rpc/proto/sync/sync.proto

@@ -0,0 +1,166 @@
+syntax = "proto3";
+
+package sync;
+
+import "google/protobuf/empty.proto";
+
+option go_package = "github.com/ava-labs/avalanchego/proto/pb/sync";
+
+// Request represents a request for information during syncing.
+message Request {
+  oneof message {
+    SyncGetRangeProofRequest range_proof_request = 1;
+    SyncGetChangeProofRequest change_proof_request = 2;
+  }
+}
+
+// The interface required by an x/sync/SyncManager for syncing.
+// Note this service definition only exists for use in tests.
+// A database shouldn't expose this over the internet, as it
+// allows for reading/writing to the database.
+service DB {
+  rpc GetMerkleRoot(google.protobuf.Empty) returns (GetMerkleRootResponse);
+
+  rpc GetProof(GetProofRequest) returns (GetProofResponse);
+
+  rpc GetChangeProof(GetChangeProofRequest) returns (GetChangeProofResponse);
+  rpc VerifyChangeProof(VerifyChangeProofRequest) returns (VerifyChangeProofResponse);
+  rpc CommitChangeProof(CommitChangeProofRequest) returns (google.protobuf.Empty);
+
+  rpc GetRangeProof(GetRangeProofRequest) returns (GetRangeProofResponse);
+  rpc CommitRangeProof(CommitRangeProofRequest) returns (google.protobuf.Empty);
+}
+
+message GetMerkleRootResponse {
+  bytes root_hash = 1;
+}
+
+message GetProofRequest {
+  bytes key = 1;
+}
+
+message GetProofResponse {
+  Proof proof = 1;
+}
+
+message Proof {
+  bytes key = 1;
+  MaybeBytes value = 2;
+  repeated ProofNode proof = 3;
+}
+
+// For use in sync client, which has a restriction on the size of
+// the response. GetChangeProof in the DB service doesn't.
+message SyncGetChangeProofRequest {
+  bytes start_root_hash = 1;
+  bytes end_root_hash = 2;
+  MaybeBytes start_key = 3;
+  MaybeBytes end_key = 4;
+  uint32 key_limit = 5;
+  uint32 bytes_limit = 6;
+}
+
+message SyncGetChangeProofResponse {
+  oneof response {
+    ChangeProof change_proof = 1;
+    RangeProof range_proof = 2;
+  }
+}
+
+message GetChangeProofRequest {
+  bytes start_root_hash = 1;
+  bytes end_root_hash = 2;
+  MaybeBytes start_key = 3;
+  MaybeBytes end_key = 4;
+  uint32 key_limit = 5;
+}
+
+message GetChangeProofResponse {
+  oneof response {
+    ChangeProof change_proof = 1;
+    // True iff server errored with merkledb.ErrInsufficientHistory.
+    bool root_not_present = 2;
+  }
+}
+
+message VerifyChangeProofRequest {
+  ChangeProof proof = 1;
+  MaybeBytes start_key = 2;
+  MaybeBytes end_key = 3;
+  bytes expected_root_hash = 4;
+}
+
+message VerifyChangeProofResponse {
+  // If empty, there was no error.
+  string error = 1;
+}
+
+message CommitChangeProofRequest {
+  ChangeProof proof = 1;
+}
+
+// For use in sync client, which has a restriction on the size of
+// the response. GetRangeProof in the DB service doesn't.
+message SyncGetRangeProofRequest {
+  bytes root_hash = 1;
+  MaybeBytes start_key = 2;
+  MaybeBytes end_key = 3;
+  uint32 key_limit = 4;
+  uint32 bytes_limit = 5;
+}
+
+message GetRangeProofRequest {
+  bytes root_hash = 1;
+  MaybeBytes start_key = 2;
+  MaybeBytes end_key = 3;
+  uint32 key_limit = 4;
+}
+
+message GetRangeProofResponse {
+  RangeProof proof = 1;
+}
+
+message CommitRangeProofRequest {
+  MaybeBytes start_key = 1;
+  RangeProof range_proof = 2;
+}
+
+message ChangeProof {
+  repeated ProofNode start_proof = 1;
+  repeated ProofNode end_proof = 2;
+  repeated KeyChange key_changes = 3;
+}
+
+message RangeProof {
+  repeated ProofNode start = 1;
+  repeated ProofNode end = 2;
+  repeated KeyValue key_values = 3;
+}
+
+message ProofNode {
+  SerializedPath key = 1;
+  MaybeBytes value_or_hash = 2;
+  map<uint32, bytes> children = 3;
+}
+
+message KeyChange {
+  bytes key = 1;
+  MaybeBytes value = 2;
+}
+
+message SerializedPath {
+  uint64 nibble_length = 1;
+  bytes value = 2;
+}
+
+message MaybeBytes {
+  bytes value = 1;
+  // If false, this is None.
+  // Otherwise this is Some.
+  bool is_nothing = 2;
+}
+
+message KeyValue {
+  bytes key = 1;
+  bytes value = 2;
+}

rpc/src/bin/client.rs

@@ -0,0 +1,3 @@
+fn main() {
+    println!("Hello from {}", file!());
+}