data(grpc): init tokenizer

inference_grpc/Cargo.toml

@@ -6,3 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+tonic = "0.10.2"
+
+[build-dependencies]
+tonic-build = { version = "0.10.2", features = ["prost"] }

inference_grpc/build.rs

@@ -0,0 +1,31 @@
+use std::env;
+use std::path::PathBuf;
+
+fn main() {
+    let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
+    tonic_build::configure()
+        .file_descriptor_set_path(out_dir.join("tokenizer_descriptor.bin"))
+        .compile(&["proto/tokenizer/tokenizer.proto"], &["proto"])
+        .unwrap();
+
+
+    build_json_codec_service();
+}
+
+fn build_json_codec_service() {
+    let encode_service = tonic_build::manual::Service::builder()
+        .name("Tokenizer")
+        .package("json.tokenizer")
+        .method(
+            tonic_build::manual::Method::builder()
+                .name("encode")
+                .route_name("Encode")
+                .input_type("crate::common::EncodeRequest")
+                .output_type("crate::common::EncodeReply")
+                .codec_path("crate::common::JsonCodec")
+                .build(),
+        )
+        .build();
+
+    tonic_build::manual::Builder::new().compile(&[encode_service]);
+}

inference_grpc/proto/tokenizer/tokenizer.proto

@@ -0,0 +1,21 @@
+syntax = "proto3";
+
+//
+//option java_multiple_files = true;
+//option java_package = "org.unitmesh.inference.tokenizer";
+//option java_outer_classname = "TokenizerProto";
+
+package tokenizer;
+
+service Tokenizer {
+  rpc Encode (EncodeRequest) returns (EncodeReply) {}
+}
+
+message EncodeRequest {
+  string text = 1;
+}
+
+// todo
+message EncodeReply {
+  string text = 1;
+}