feat: add better message when version mismatch detected

src/lib.rs

@@ -3,6 +3,8 @@ use std::path::PathBuf;
 use std::ptr;
 use tokenizers::tokenizer::Tokenizer;
 
+const CARGO_PKG_VERSION: &str = env!("CARGO_PKG_VERSION");
+
 #[repr(C)]
 pub struct TokenizerOptions {
     encode_special_tokens: bool,
@@ -19,6 +21,11 @@ pub struct Buffer {
     len: usize,
 }
 
+#[no_mangle]
+pub extern "C" fn version() -> *const libc::c_char {
+    std::ffi::CString::new(CARGO_PKG_VERSION).unwrap().into_raw()
+}
+
 #[no_mangle]
 pub extern "C" fn from_bytes(bytes: *const u8, len: u32, opts: &TokenizerOptions) -> *mut Tokenizer {
     let bytes_slice = unsafe { std::slice::from_raw_parts(bytes, len as usize) };

tokenizer.go

@@ -11,10 +11,21 @@ import "C"
 
 // NOTE: There should be NO space between the comments and the `import "C"` line.
 import (
+	"fmt"
 	"io"
 	"unsafe"
 )
 
+const WANT_VERSION = "0.20.2"
+
+func init() {
+	version := C.version()
+	got := C.GoString(version)
+	if got != WANT_VERSION {
+		panic(fmt.Errorf("tokenizers library version mismatch, want: %s, got: %s", WANT_VERSION, got))
+	}
+}
+
 type Tokenizer struct {
 	tokenizer unsafe.Pointer
 }

tokenizers.h

@@ -24,6 +24,8 @@ struct Buffer {
   uint32_t len;
 };
 
+const char *version();
+
 void *from_bytes(const uint8_t *config, uint32_t len, const struct TokenizerOptions *options);
 
 void *from_bytes_with_truncation(const uint8_t *config, uint32_t len, uint32_t max_len, uint8_t direction);