feat: add Czech stemmer and stopwords

Cargo.toml

@@ -38,7 +38,7 @@ levenshtein_automata = "0.2.1"
 uuid = { version = "1.0.0", features = ["v4", "serde"] }
 crossbeam-channel = "0.5.4"
 rust-stemmers = { version = "1.2.0", optional = true }
-tantivy-stemmers = { version = "0.4.0", default-features = false, features = ["polish_yarovoy"] }
+tantivy-stemmers = { version = "0.4.0", default-features = false, features = ["czech_dolamic_aggressive", "polish_yarovoy"] }
 downcast-rs = "2.0.1"
 bitpacking = { version = "0.9.3", default-features = false, features = [
     "bitpacker4x",

src/tokenizer/mod.rs

@@ -263,6 +263,36 @@ pub(crate) mod tests {
         assert_token(&tokens[2], 2, "podatnik", 22, 31);
     }
 
+    #[cfg(feature = "stemmer")]
+    #[test]
+    fn test_cs_stemmer() {
+        let tokenizer_manager = TokenizerManager::default();
+        tokenizer_manager.register(
+            "cs_stem",
+            TextAnalyzer::builder(SimpleTokenizer::default())
+                .filter(RemoveLongFilter::limit(40))
+                .filter(LowerCaser)
+                .filter(Stemmer::new(Language::Czech))
+                .build(),
+        );
+
+        let mut cs_tokenizer = tokenizer_manager.get("cs_stem").unwrap();
+        let mut tokens: Vec<Token> = vec![];
+        {
+            let mut add_token = |token: &Token| {
+                tokens.push(token.clone());
+            };
+            cs_tokenizer
+                .token_stream("Novinka počasí funguje.")
+                .process(&mut add_token);
+        }
+
+        assert_eq!(tokens.len(), 3);
+        assert_token(&tokens[0], 0, "novink", 0, 7);
+        assert_token(&tokens[1], 1, "počas", 8, 16);
+        assert_token(&tokens[2], 2, "funguj", 17, 24);
+    }
+
     #[cfg(feature = "stemmer")]
     #[test]
     fn test_non_en_stemmer() {

src/tokenizer/stemmer.rs

@@ -16,6 +16,7 @@ enum StemmerAlgorithm {
 #[allow(missing_docs)]
 pub enum Language {
     Arabic,
+    Czech,
     Danish,
     Dutch,
     English,
@@ -41,6 +42,9 @@ impl Language {
         use self::Language::*;
         match self {
             Arabic => StemmerAlgorithm::Rust(rust_stemmers::Algorithm::Arabic),
+            Czech => {
+                StemmerAlgorithm::Tantivy(tantivy_stemmers::algorithms::czech_dolamic_aggressive)
+            }
             Danish => StemmerAlgorithm::Rust(rust_stemmers::Algorithm::Danish),
             Dutch => StemmerAlgorithm::Rust(rust_stemmers::Algorithm::Dutch),
             English => StemmerAlgorithm::Rust(rust_stemmers::Algorithm::English),

src/tokenizer/stop_word_filter/mod.rs

@@ -36,6 +36,7 @@ impl StopWordFilter {
     #[cfg(feature = "stopwords")]
     pub fn new(language: Language) -> Option<Self> {
         let words = match language {
+            Language::Czech => stopwords::CZECH,
             Language::Danish => stopwords::DANISH,
             Language::Dutch => stopwords::DUTCH,
             Language::English => {
@@ -133,6 +134,8 @@ impl<T: TokenStream> TokenStream for StopWordFilterStream<T> {
 #[cfg(test)]
 mod tests {
     use crate::tokenizer::tests::assert_token;
+    #[cfg(all(feature = "stemmer", feature = "stopwords"))]
+    use crate::tokenizer::Language;
     use crate::tokenizer::{SimpleTokenizer, StopWordFilter, TextAnalyzer, Token};
 
     #[test]
@@ -164,4 +167,22 @@ mod tests {
         token_stream.process(&mut add_token);
         tokens
     }
+
+    #[cfg(all(feature = "stemmer", feature = "stopwords"))]
+    #[test]
+    fn test_stop_word_czech_language() {
+        let mut analyzer = TextAnalyzer::builder(SimpleTokenizer::default())
+            .filter(StopWordFilter::new(Language::Czech).expect("Czech stopwords should exist"))
+            .build();
+        let mut token_stream = analyzer.token_stream("bez jablek a vody");
+        let mut tokens: Vec<Token> = vec![];
+        let mut add_token = |token: &Token| {
+            tokens.push(token.clone());
+        };
+        token_stream.process(&mut add_token);
+
+        assert_eq!(tokens.len(), 2);
+        assert_token(&tokens[0], 1, "jablek", 4, 10);
+        assert_token(&tokens[1], 3, "vody", 13, 17);
+    }
 }