diff --git a/CHANGES.md b/CHANGES.md index 25473dd..4c4f494 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,6 +2,9 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## 0.13.2 (2022-04-08) +- Update Lindera to 0.13.2 #48 @mosuka + ## 0.13.1 (2022-04-08) - Update Lindera to 0.13.1 #47 @mosuka diff --git a/Cargo.toml b/Cargo.toml index ee40470..975afa7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lindera-tantivy" -version = "0.13.1" +version = "0.13.2" edition = "2021" description = "Lindera Tokenizer for Tantivy." documentation = "https://docs.rs/lindera-tantivy" @@ -22,7 +22,7 @@ cc-cedict = ["lindera/cc-cedict"] # Chinese dictionary [dependencies] tantivy = "0.17" -lindera = "0.13.1" +lindera = "0.13.2" [dev-dependencies] criterion = "0.3" diff --git a/benches/bench.rs b/benches/bench.rs index 2d1d0b5..186478e 100644 --- a/benches/bench.rs +++ b/benches/bench.rs @@ -7,9 +7,9 @@ fn bench_indexing(c: &mut Criterion) { use tantivy::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions}; use tantivy::Index; - use lindera::mode::{Mode, Penalty}; - use lindera::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; + use lindera_tantivy::mode::{Mode, Penalty}; use lindera_tantivy::tokenizer::LinderaTokenizer; + use lindera_tantivy::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; // create schema builder let mut schema_builder = Schema::builder(); diff --git a/examples/cc-cedict_example.rs b/examples/cc-cedict_example.rs index d8218cf..2b0ffeb 100644 --- a/examples/cc-cedict_example.rs +++ b/examples/cc-cedict_example.rs @@ -6,9 +6,9 @@ fn main() -> tantivy::Result<()> { use tantivy::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions}; use tantivy::Index; - use lindera::mode::{Mode, Penalty}; - use lindera::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; + use lindera_tantivy::mode::{Mode, Penalty}; use lindera_tantivy::tokenizer::LinderaTokenizer; + use lindera_tantivy::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; // create schema builder let mut schema_builder = Schema::builder(); diff --git a/examples/ipadic_example.rs b/examples/ipadic_example.rs index 7d87c00..e005368 100644 --- a/examples/ipadic_example.rs +++ b/examples/ipadic_example.rs @@ -6,9 +6,9 @@ fn main() -> tantivy::Result<()> { use tantivy::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions}; use tantivy::Index; - use lindera::mode::{Mode, Penalty}; - use lindera::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; + use lindera_tantivy::mode::{Mode, Penalty}; use lindera_tantivy::tokenizer::LinderaTokenizer; + use lindera_tantivy::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; // create schema builder let mut schema_builder = Schema::builder(); diff --git a/examples/ko-dic_example.rs b/examples/ko-dic_example.rs index d3a95f1..5db3fd2 100644 --- a/examples/ko-dic_example.rs +++ b/examples/ko-dic_example.rs @@ -6,9 +6,9 @@ fn main() -> tantivy::Result<()> { use tantivy::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions}; use tantivy::Index; - use lindera::mode::{Mode, Penalty}; - use lindera::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; + use lindera_tantivy::mode::{Mode, Penalty}; use lindera_tantivy::tokenizer::LinderaTokenizer; + use lindera_tantivy::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; // create schema builder let mut schema_builder = Schema::builder(); diff --git a/examples/unidic_example.rs b/examples/unidic_example.rs index 6750e3a..1b0542b 100644 --- a/examples/unidic_example.rs +++ b/examples/unidic_example.rs @@ -6,9 +6,9 @@ fn main() -> tantivy::Result<()> { use tantivy::schema::{IndexRecordOption, Schema, TextFieldIndexing, TextOptions}; use tantivy::Index; - use lindera::mode::{Mode, Penalty}; - use lindera::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; + use lindera_tantivy::mode::{Mode, Penalty}; use lindera_tantivy::tokenizer::LinderaTokenizer; + use lindera_tantivy::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; // create schema builder let mut schema_builder = Schema::builder(); diff --git a/src/lib.rs b/src/lib.rs index d5f84ee..e1a2278 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,7 @@ +pub mod mode; pub mod stream; pub mod tokenizer; + +use lindera::LinderaResult as LLinderaResult; + +pub type LinderaResult = LLinderaResult; diff --git a/src/mode.rs b/src/mode.rs new file mode 100644 index 0000000..8836ee9 --- /dev/null +++ b/src/mode.rs @@ -0,0 +1,4 @@ +use lindera::mode::{Mode as LinderaMode, Penalty as LinderaPenalty}; + +pub type Penalty = LinderaPenalty; +pub type Mode = LinderaMode; diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 9602e13..e8b13c7 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,11 +1,17 @@ -use tantivy::tokenizer::{BoxTokenStream, Tokenizer}; +use tantivy::tokenizer::{BoxTokenStream, Tokenizer as TTokenizer}; -use lindera::tokenizer::{Tokenizer as LTokenizer, TokenizerConfig}; -use lindera::LinderaResult; +use lindera::tokenizer::{ + DictionaryType as LDictionaryType, Tokenizer as LTokenizer, + TokenizerConfig as LTokenizerConfig, UserDictionaryType as LUserDictionaryType, +}; use crate::stream::LinderaTokenStream; +use crate::LinderaResult; + +pub type DictionaryType = LDictionaryType; +pub type UserDictionaryType = LUserDictionaryType; +pub type TokenizerConfig = LTokenizerConfig; -/// Tokenize text with the specified mode and dictionary. pub struct LinderaTokenizer { pub tokenizer: LTokenizer, } @@ -32,7 +38,7 @@ impl LinderaTokenizer { } } -impl Tokenizer for LinderaTokenizer { +impl TTokenizer for LinderaTokenizer { fn token_stream<'a>(&self, text: &'a str) -> BoxTokenStream<'a> { let result = match self.tokenizer.tokenize(text) { Ok(result) => result, @@ -53,10 +59,8 @@ impl Tokenizer for LinderaTokenizer { mod tests { use tantivy::tokenizer::{BoxTokenStream, Token, Tokenizer}; - use lindera::mode::{Mode, Penalty}; - use lindera::tokenizer::{DictionaryType, TokenizerConfig, UserDictionaryType}; - - use crate::tokenizer::LinderaTokenizer; + use crate::mode::{Mode, Penalty}; + use crate::tokenizer::{DictionaryType, LinderaTokenizer, TokenizerConfig, UserDictionaryType}; fn test_helper(mut tokenizer: BoxTokenStream) -> Vec { let mut tokens: Vec = vec![];