Update tokenizers to 0.15.0

mix.exs

@@ -2,7 +2,7 @@ defmodule Tokenizers.MixProject do
   use Mix.Project
 
   @source_url "https://github.com/elixir-nx/tokenizers"
-  @version "0.4.0"
+  @version "0.5.0-dev"
 
   def project do
     [

native/ex_tokenizers/Cargo.toml

@@ -13,5 +13,5 @@ crate-type = ["cdylib"]
 anyhow = "1"
 rustler = "0.29.1"
 thiserror = "1"
-tokenizers = { version = "0.13.3", default-features = false, features = ["onig", "esaxx_fast"]}
+tokenizers = { version = "0.15.0", default-features = false, features = ["onig", "esaxx_fast"]}
 serde = { version = "1.0", features = [ "rc", "derive" ] }

native/ex_tokenizers/src/models.rs

@@ -378,7 +378,7 @@ pub fn models_unigram_init(
     };
 
     Ok(ExTokenizersModel::new(
-        tokenizers::models::unigram::Unigram::from(vocab, unk_id)?,
+        tokenizers::models::unigram::Unigram::from(vocab, unk_id, false)?,
     ))
 }
 

native/ex_tokenizers/src/tokenizer.rs

@@ -125,7 +125,7 @@ fn apply_load_options(mut tokenizer: ExTokenizerImpl, options: Vec<LoadOption>)
     }
 
     if opts.disable_truncation {
-        tokenizer.with_truncation(None);
+        tokenizer.with_truncation(None).unwrap();
     }
 
     tokenizer
@@ -335,14 +335,14 @@ pub fn tokenizer_set_truncation(
         TruncationOption::Direction(direction) => truncation.direction = direction.into(),
     });
     let mut new_tokenizer = tokenizer.resource.0.clone();
-    new_tokenizer.with_truncation(Some(truncation));
+    new_tokenizer.with_truncation(Some(truncation)).unwrap();
     new_tokenizer.into()
 }
 
 #[rustler::nif]
 pub fn tokenizer_disable_truncation(tokenizer: ExTokenizersTokenizer) -> ExTokenizersTokenizer {
     let mut new_tokenizer = tokenizer.resource.0.clone();
-    new_tokenizer.with_truncation(None);
+    new_tokenizer.with_truncation(None).unwrap();
     new_tokenizer.into()
 }
 
@@ -530,7 +530,10 @@ pub fn tokenizer_decode(
         }
     });
 
-    Ok(tokenizer.resource.0.decode(ids, opts.skip_special_tokens)?)
+    Ok(tokenizer
+        .resource
+        .0
+        .decode(&ids, opts.skip_special_tokens)?)
 }
 
 #[rustler::nif(schedule = "DirtyCpu")]
@@ -551,10 +554,14 @@ pub fn tokenizer_decode_batch(
         }
     });
 
-    Ok(tokenizer
-        .resource
-        .0
-        .decode_batch(sentences, opts.skip_special_tokens)?)
+    Ok(tokenizer.resource.0.decode_batch(
+        sentences
+            .iter()
+            .map(Vec::as_slice)
+            .collect::<Vec<&[u32]>>()
+            .as_slice(),
+        opts.skip_special_tokens,
+    )?)
 }
 
 #[rustler::nif]