feat: better error message when tokenizers lib mismatch

BUILD.bazel

@@ -23,6 +23,7 @@ rust_static_library(
     srcs = glob([
         "src/**/*.rs",
     ]),
+    version = "1.20.2",
     aliases = aliases(),
     proc_macro_deps = all_crate_deps(
         proc_macro = True,

Cargo.Bazel.lock

@@ -1,5 +1,5 @@
 {
-  "checksum": "274eda904d8c182522e7565c734cf8c608671ee2bdb346dccb07c10d44904563",
+  "checksum": "8c24cec6818168bbb4b50e04f24841ad2a219f25d0f3aab050fcb832fef53eef",
   "crates": {
     "aho-corasick 1.1.2": {
       "name": "aho-corasick",
@@ -4486,13 +4486,13 @@
       },
       "license": "Apache-2.0 OR MIT"
     },
-    "tokenizers 0.20.0": {
+    "tokenizers 0.20.2": {
       "name": "tokenizers",
-      "version": "0.20.0",
+      "version": "0.20.2",
       "repository": {
         "Http": {
-          "url": "https://static.crates.io/crates/tokenizers/0.20.0/download",
-          "sha256": "c8a24d7f7d6be5b9d1377418b893ab1808af0074f5d1bb2c64784452ddd2aa70"
+          "url": "https://static.crates.io/crates/tokenizers/0.20.2/download",
+          "sha256": "fef9821871de9277bd7d8a7568e798e775db7eede56e290b4b03f1dfb9ac6769"
         }
       },
       "targets": [
@@ -4628,13 +4628,13 @@
           ],
           "selects": {}
         },
-        "version": "0.20.0"
+        "version": "0.20.2"
       },
       "license": "Apache-2.0"
     },
-    "tokenizers 0.9.0": {
+    "tokenizers 1.20.2": {
       "name": "tokenizers",
-      "version": "0.9.0",
+      "version": "1.20.2",
       "repository": null,
       "targets": [],
       "library_target_name": null,
@@ -4649,7 +4649,7 @@
               "target": "libc"
             },
             {
-              "id": "tokenizers 0.20.0",
+              "id": "tokenizers 0.20.2",
               "target": "tokenizers"
             }
           ],
@@ -4669,7 +4669,7 @@
           "selects": {}
         },
         "edition": "2021",
-        "version": "0.9.0"
+        "version": "1.20.2"
       },
       "license": null
     },
@@ -6375,7 +6375,7 @@
   },
   "binary_crates": [],
   "workspace_members": {
-    "tokenizers 0.9.0": ""
+    "tokenizers 1.20.2": ""
   },
   "conditions": {
     "aarch64-apple-darwin": [

Cargo.toml

@@ -1,14 +1,16 @@
 [package]
-name = "tokenizers" 
-version = "0.9.0"
+name = "tokenizers"
+# Bump major version every time we change the behavior of the library.
+# Bump minor.patch version every time we bump tokenizers dependency version.
+version = "1.20.2"
 edition = "2021"
 
 [lib]
 crate-type = ["staticlib"]
 
 [dependencies]
 libc = "0.2.140"
-tokenizers = {version = "0.20.0" }
+tokenizers = {version = "0.20.2" }
 
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["html_reports"] }

src/lib.rs

@@ -3,6 +3,8 @@ use std::path::PathBuf;
 use std::ptr;
 use tokenizers::tokenizer::Tokenizer;
 
+const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION");
+
 #[repr(C)]
 pub struct TokenizerOptions {
     encode_special_tokens: bool,
@@ -19,6 +21,11 @@ pub struct Buffer {
     len: usize,
 }
 
+#[no_mangle]
+pub extern "C" fn version() -> *const libc::c_char {
+    std::ffi::CString::new(CARGO_PKG_VERSION).unwrap().into_raw()
+}
+
 #[no_mangle]
 pub extern "C" fn from_bytes(bytes: *const u8, len: u32, opts: &TokenizerOptions) -> *mut Tokenizer {
     let bytes_slice = unsafe { std::slice::from_raw_parts(bytes, len as usize) };

tokenizer.go

@@ -11,10 +11,21 @@ import "C"
 
 // NOTE: There should be NO space between the comments and the `import "C"` line.
 import (
+	"fmt"
 	"io"
 	"unsafe"
 )
 
+const WANT_VERSION = "1.20.2"
+
+func init() {
+	version := C.version()
+	got := C.GoString(version)
+	if got != WANT_VERSION {
+		panic(fmt.Errorf("tokenizers library version mismatch, want: %s, got: %s", WANT_VERSION, got))
+	}
+}
+
 type Tokenizer struct {
 	tokenizer unsafe.Pointer
 }

tokenizers.h

@@ -24,6 +24,8 @@ struct Buffer {
   uint32_t len;
 };
 
+const char *version();
+
 void *from_bytes(const uint8_t *config, uint32_t len, const struct TokenizerOptions *options);
 
 void *from_bytes_with_truncation(const uint8_t *config, uint32_t len, uint32_t max_len, uint8_t direction);