fix cross compilation: libc::c_char is platform dependent

Makefile

@@ -18,7 +18,7 @@ release-darwin-%:
 release-linux-%:
 	docker buildx build --platform linux/$* -f release/Dockerfile . -t tokenizers.linux-$*
 	mkdir -p artifacts/linux-$*
-	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint cp tokenizers.linux-$* /go/pkg/mod/github.com/daulet/tokenizers@v0.6.0/libtokenizers.a /mnt/libtokenizers.a
+	docker run -v $(PWD)/artifacts/linux-$*:/mnt --entrypoint cp tokenizers.linux-$* /workspace/tokenizers/libtokenizers.a /mnt/libtokenizers.a
 	cd artifacts/linux-$* && \
 		tar -czf libtokenizers.linux-$*.tar.gz libtokenizers.a
 	mkdir -p artifacts/all

release/Dockerfile

@@ -11,10 +11,13 @@ RUN cargo build --release
 FROM golang:1.21 as builder-go
 ARG TARGETPLATFORM
 WORKDIR /workspace
-COPY ./release .
-COPY ./test/data ./test/data
-RUN go mod download
+COPY ./release/go.mod   .
+COPY ./release/main.go  .
+# can't rely on tokenizers module because latest release doesn't include recent changes
+COPY tokenizer.go       ./tokenizers/
+COPY tokenizers.h       ./tokenizers/
 COPY --from=builder-rust \
     /workspace/target/release/libtokenizers.a \
-    /go/pkg/mod/github.com/daulet/tokenizers@v0.6.0/libtokenizers.a
-RUN go run main.go
+    ./tokenizers/
+COPY ./test/data        ./test/data
+RUN go run .

release/go.mod

@@ -1,5 +1,3 @@
-module github.com/daulet/tokenizers/example
+module github.com/daulet/tokenizers/release
 
-go 1.21
-
-require github.com/daulet/tokenizers v0.6.0
+go 1.21.5

release/main.go

@@ -3,7 +3,7 @@ package main
 import (
 	"fmt"
 
-	"github.com/daulet/tokenizers"
+	"github.com/daulet/tokenizers/release/tokenizers"
 )
 
 func main() {

src/lib.rs

@@ -90,7 +90,7 @@ pub extern "C" fn encode(ptr: *mut libc::c_void, message: *const libc::c_char, o
         std::mem::forget(vec_type_ids);
     }
 
-    let mut tokens: *mut *mut i8 = ptr::null_mut();
+    let mut tokens: *mut *mut libc::c_char = ptr::null_mut();
     if options.return_tokens {
         let mut vec_tokens = encoding.get_tokens()
             .to_vec().into_iter()
@@ -180,7 +180,7 @@ pub extern "C" fn free_buffer(buf: Buffer) {
         unsafe {
             let strings = Vec::from_raw_parts(buf.tokens, buf.len, buf.len);
             for s in strings {
-                drop(std::ffi::CString::from_raw(s));
+                drop(std::ffi::CString::from_raw(s.cast::<libc::c_char>()));
             }   
         }
     }