From ab39b370695c5f085072e6fe2a0ff1b093a7adbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Jan=C4=8Da=C5=99=C3=ADk?= Date: Thu, 5 Mar 2026 17:00:26 +0100 Subject: [PATCH 1/4] feat: add Czech stemmer and stopwords --- Cargo.toml | 2 +- src/tokenizer/mod.rs | 30 ++ src/tokenizer/stemmer.rs | 4 + src/tokenizer/stop_word_filter/mod.rs | 21 +- src/tokenizer/stop_word_filter/stopwords.rs | 426 ++++++++++++++++++++ 5 files changed, 481 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 87f87d426e..7525175566 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ levenshtein_automata = "0.2.1" uuid = { version = "1.0.0", features = ["v4", "serde"] } crossbeam-channel = "0.5.4" rust-stemmers = { version = "1.2.0", optional = true } -tantivy-stemmers = { version = "0.4.0", default-features = false, features = ["polish_yarovoy"] } +tantivy-stemmers = { version = "0.4.0", default-features = false, features = ["czech_dolamic_aggressive", "polish_yarovoy"] } downcast-rs = "2.0.1" bitpacking = { version = "0.9.3", default-features = false, features = [ "bitpacker4x", diff --git a/src/tokenizer/mod.rs b/src/tokenizer/mod.rs index 4c415d926b..81ba5e449a 100644 --- a/src/tokenizer/mod.rs +++ b/src/tokenizer/mod.rs @@ -263,6 +263,36 @@ pub(crate) mod tests { assert_token(&tokens[2], 2, "podatnik", 22, 31); } + #[cfg(feature = "stemmer")] + #[test] + fn test_cs_stemmer() { + let tokenizer_manager = TokenizerManager::default(); + tokenizer_manager.register( + "cs_stem", + TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(RemoveLongFilter::limit(40)) + .filter(LowerCaser) + .filter(Stemmer::new(Language::Czech)) + .build(), + ); + + let mut cs_tokenizer = tokenizer_manager.get("cs_stem").unwrap(); + let mut tokens: Vec = vec![]; + { + let mut add_token = |token: &Token| { + tokens.push(token.clone()); + }; + cs_tokenizer + .token_stream("Novinka počasí funguje.") + .process(&mut add_token); + } + + assert_eq!(tokens.len(), 3); + assert_token(&tokens[0], 0, "novink", 0, 7); + assert_token(&tokens[1], 1, "počas", 8, 16); + assert_token(&tokens[2], 2, "funguj", 17, 24); + } + #[cfg(feature = "stemmer")] #[test] fn test_non_en_stemmer() { diff --git a/src/tokenizer/stemmer.rs b/src/tokenizer/stemmer.rs index a02431f5ae..d66617b5b8 100644 --- a/src/tokenizer/stemmer.rs +++ b/src/tokenizer/stemmer.rs @@ -16,6 +16,7 @@ enum StemmerAlgorithm { #[allow(missing_docs)] pub enum Language { Arabic, + Czech, Danish, Dutch, English, @@ -41,6 +42,9 @@ impl Language { use self::Language::*; match self { Arabic => StemmerAlgorithm::Rust(rust_stemmers::Algorithm::Arabic), + Czech => { + StemmerAlgorithm::Tantivy(tantivy_stemmers::algorithms::czech_dolamic_aggressive) + } Danish => StemmerAlgorithm::Rust(rust_stemmers::Algorithm::Danish), Dutch => StemmerAlgorithm::Rust(rust_stemmers::Algorithm::Dutch), English => StemmerAlgorithm::Rust(rust_stemmers::Algorithm::English), diff --git a/src/tokenizer/stop_word_filter/mod.rs b/src/tokenizer/stop_word_filter/mod.rs index 6bdb212e4f..b21d5602c9 100644 --- a/src/tokenizer/stop_word_filter/mod.rs +++ b/src/tokenizer/stop_word_filter/mod.rs @@ -36,6 +36,7 @@ impl StopWordFilter { #[cfg(feature = "stopwords")] pub fn new(language: Language) -> Option { let words = match language { + Language::Czech => stopwords::CZECH, Language::Danish => stopwords::DANISH, Language::Dutch => stopwords::DUTCH, Language::English => { @@ -133,7 +134,7 @@ impl TokenStream for StopWordFilterStream { #[cfg(test)] mod tests { use crate::tokenizer::tests::assert_token; - use crate::tokenizer::{SimpleTokenizer, StopWordFilter, TextAnalyzer, Token}; + use crate::tokenizer::{Language, SimpleTokenizer, StopWordFilter, TextAnalyzer, Token}; #[test] fn test_stop_word() { @@ -164,4 +165,22 @@ mod tests { token_stream.process(&mut add_token); tokens } + + #[cfg(feature = "stopwords")] + #[test] + fn test_stop_word_czech_language() { + let mut analyzer = TextAnalyzer::builder(SimpleTokenizer::default()) + .filter(StopWordFilter::new(Language::Czech).expect("Czech stopwords should exist")) + .build(); + let mut token_stream = analyzer.token_stream("bez jablek a vody"); + let mut tokens: Vec = vec![]; + let mut add_token = |token: &Token| { + tokens.push(token.clone()); + }; + token_stream.process(&mut add_token); + + assert_eq!(tokens.len(), 2); + assert_token(&tokens[0], 1, "jablek", 4, 10); + assert_token(&tokens[1], 3, "vody", 13, 17); + } } diff --git a/src/tokenizer/stop_word_filter/stopwords.rs b/src/tokenizer/stop_word_filter/stopwords.rs index 9ee2dfc7a8..d252868203 100644 --- a/src/tokenizer/stop_word_filter/stopwords.rs +++ b/src/tokenizer/stop_word_filter/stopwords.rs @@ -1521,6 +1521,432 @@ pub const NORWEGIAN: &[&str] = &[ "vart", ]; +// Source: https://github.com/stopwords-iso/stopwords-cs (MIT license) +pub const CZECH: &[&str] = &[ + "a", + "aby", + "ahoj", + "aj", + "ale", + "anebo", + "ani", + "aniž", + "ano", + "asi", + "aspoň", + "atd", + "atp", + "az", + "ačkoli", + "až", + "bez", + "beze", + "blízko", + "bohužel", + "brzo", + "bude", + "budem", + "budeme", + "budes", + "budete", + "budeš", + "budou", + "budu", + "by", + "byl", + "byla", + "byli", + "bylo", + "byly", + "bys", + "byt", + "být", + "během", + "chce", + "chceme", + "chcete", + "chceš", + "chci", + "chtít", + "chtějí", + "chut'", + "chuti", + "ci", + "clanek", + "clanku", + "clanky", + "co", + "coz", + "což", + "cz", + "daleko", + "dalsi", + "další", + "den", + "deset", + "design", + "devatenáct", + "devět", + "dnes", + "do", + "dobrý", + "docela", + "dva", + "dvacet", + "dvanáct", + "dvě", + "dál", + "dále", + "děkovat", + "děkujeme", + "děkuji", + "email", + "ho", + "hodně", + "i", + "jak", + "jakmile", + "jako", + "jakož", + "jde", + "je", + "jeden", + "jedenáct", + "jedna", + "jedno", + "jednou", + "jedou", + "jeho", + "jehož", + "jej", + "jeji", + "jejich", + "její", + "jelikož", + "jemu", + "jen", + "jenom", + "jenž", + "jeste", + "jestli", + "jestliže", + "ještě", + "jež", + "ji", + "jich", + "jimi", + "jinak", + "jine", + "jiné", + "jiz", + "již", + "jsem", + "jses", + "jseš", + "jsi", + "jsme", + "jsou", + "jste", + "já", + "jí", + "jím", + "jíž", + "jšte", + "k", + "kam", + "každý", + "kde", + "kdo", + "kdy", + "kdyz", + "když", + "ke", + "kolik", + "kromě", + "ktera", + "ktere", + "kteri", + "kterou", + "ktery", + "která", + "které", + "který", + "kteři", + "kteří", + "ku", + "kvůli", + "ma", + "mají", + "mate", + "me", + "mezi", + "mi", + "mit", + "mne", + "mnou", + "mně", + "moc", + "mohl", + "mohou", + "moje", + "moji", + "možná", + "muj", + "musí", + "muze", + "my", + "má", + "málo", + "mám", + "máme", + "máte", + "máš", + "mé", + "mí", + "mít", + "mě", + "můj", + "může", + "na", + "nad", + "nade", + "nam", + "napiste", + "napište", + "naproti", + "nas", + "nasi", + "načež", + "naše", + "naši", + "ne", + "nebo", + "nebyl", + "nebyla", + "nebyli", + "nebyly", + "nechť", + "nedělají", + "nedělá", + "nedělám", + "neděláme", + "neděláte", + "neděláš", + "neg", + "nejsi", + "nejsou", + "nemají", + "nemáme", + "nemáte", + "neměl", + "neni", + "není", + "nestačí", + "nevadí", + "nez", + "než", + "nic", + "nich", + "nimi", + "nove", + "novy", + "nové", + "nový", + "nula", + "ná", + "nám", + "námi", + "nás", + "náš", + "ní", + "ním", + "ně", + "něco", + "nějak", + "někde", + "někdo", + "němu", + "němuž", + "o", + "od", + "ode", + "on", + "ona", + "oni", + "ono", + "ony", + "osm", + "osmnáct", + "pak", + "patnáct", + "po", + "pod", + "podle", + "pokud", + "potom", + "pouze", + "pozdě", + "pořád", + "prave", + "pravé", + "pred", + "pres", + "pri", + "pro", + "proc", + "prostě", + "prosím", + "proti", + "proto", + "protoze", + "protože", + "proč", + "prvni", + "první", + "práve", + "pta", + "pět", + "před", + "přede", + "přes", + "přese", + "při", + "přičemž", + "re", + "rovně", + "s", + "se", + "sedm", + "sedmnáct", + "si", + "sice", + "skoro", + "smí", + "smějí", + "snad", + "spolu", + "sta", + "sto", + "strana", + "sté", + "sve", + "svych", + "svym", + "svymi", + "své", + "svých", + "svým", + "svými", + "svůj", + "ta", + "tady", + "tak", + "take", + "takhle", + "taky", + "takze", + "také", + "takže", + "tam", + "tamhle", + "tamhleto", + "tamto", + "tato", + "te", + "tebe", + "tebou", + "ted'", + "tedy", + "tema", + "ten", + "tento", + "teto", + "ti", + "tim", + "timto", + "tipy", + "tisíc", + "tisíce", + "to", + "tobě", + "tohle", + "toho", + "tohoto", + "tom", + "tomto", + "tomu", + "tomuto", + "toto", + "trošku", + "tu", + "tuto", + "tvoje", + "tvá", + "tvé", + "tvůj", + "ty", + "tyto", + "téma", + "této", + "tím", + "tímto", + "tě", + "těm", + "těma", + "těmu", + "třeba", + "tři", + "třináct", + "u", + "určitě", + "uz", + "už", + "v", + "vam", + "vas", + "vase", + "vaše", + "vaši", + "ve", + "vedle", + "večer", + "vice", + "vlastně", + "vsak", + "vy", + "vám", + "vámi", + "vás", + "váš", + "více", + "však", + "všechen", + "všechno", + "všichni", + "vůbec", + "vždy", + "z", + "za", + "zatímco", + "zač", + "zda", + "zde", + "ze", + "zpet", + "zpravy", + "zprávy", + "zpět", + "čau", + "či", + "článek", + "článku", + "články", + "čtrnáct", + "čtyři", + "šest", + "šestnáct", + "že",]; + // Source: https://github.com/stopwords-iso/stopwords-pl (MIT license) pub const POLISH: &[&str] = &[ "a", From 36e791ce18c267b43a00926c3d3d6e7a30e90aca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Jan=C4=8Da=C5=99=C3=ADk?= Date: Fri, 6 Mar 2026 16:22:32 +0100 Subject: [PATCH 2/4] test: gate czech stopword test on stemmer feature --- src/tokenizer/stop_word_filter/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tokenizer/stop_word_filter/mod.rs b/src/tokenizer/stop_word_filter/mod.rs index b21d5602c9..32d8331e94 100644 --- a/src/tokenizer/stop_word_filter/mod.rs +++ b/src/tokenizer/stop_word_filter/mod.rs @@ -134,7 +134,9 @@ impl TokenStream for StopWordFilterStream { #[cfg(test)] mod tests { use crate::tokenizer::tests::assert_token; - use crate::tokenizer::{Language, SimpleTokenizer, StopWordFilter, TextAnalyzer, Token}; + use crate::tokenizer::{SimpleTokenizer, StopWordFilter, TextAnalyzer, Token}; + #[cfg(all(feature = "stemmer", feature = "stopwords"))] + use crate::tokenizer::Language; #[test] fn test_stop_word() { @@ -166,7 +168,7 @@ mod tests { tokens } - #[cfg(feature = "stopwords")] + #[cfg(all(feature = "stemmer", feature = "stopwords"))] #[test] fn test_stop_word_czech_language() { let mut analyzer = TextAnalyzer::builder(SimpleTokenizer::default()) From c5ad39759b96498dbe32cc834d87e2d14c8a8a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Jan=C4=8Da=C5=99=C3=ADk?= Date: Fri, 6 Mar 2026 16:27:23 +0100 Subject: [PATCH 3/4] style: fix rustfmt import order in stopword tests --- src/tokenizer/stop_word_filter/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tokenizer/stop_word_filter/mod.rs b/src/tokenizer/stop_word_filter/mod.rs index 32d8331e94..21fe1f422d 100644 --- a/src/tokenizer/stop_word_filter/mod.rs +++ b/src/tokenizer/stop_word_filter/mod.rs @@ -134,9 +134,9 @@ impl TokenStream for StopWordFilterStream { #[cfg(test)] mod tests { use crate::tokenizer::tests::assert_token; - use crate::tokenizer::{SimpleTokenizer, StopWordFilter, TextAnalyzer, Token}; #[cfg(all(feature = "stemmer", feature = "stopwords"))] use crate::tokenizer::Language; + use crate::tokenizer::{SimpleTokenizer, StopWordFilter, TextAnalyzer, Token}; #[test] fn test_stop_word() { From fc4c629cdf5e5ed371e2671596de9f3bd1853c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jon=C3=A1=C5=A1=20Jan=C4=8Da=C5=99=C3=ADk?= Date: Thu, 19 Mar 2026 22:10:50 +0100 Subject: [PATCH 4/4] docs: pin Czech stopwords source permalink --- src/tokenizer/stop_word_filter/stopwords.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tokenizer/stop_word_filter/stopwords.rs b/src/tokenizer/stop_word_filter/stopwords.rs index d252868203..af237afb23 100644 --- a/src/tokenizer/stop_word_filter/stopwords.rs +++ b/src/tokenizer/stop_word_filter/stopwords.rs @@ -1521,7 +1521,7 @@ pub const NORWEGIAN: &[&str] = &[ "vart", ]; -// Source: https://github.com/stopwords-iso/stopwords-cs (MIT license) +// Source: https://github.com/stopwords-iso/stopwords-cs/blob/08a7a251f35c8ef6fd666513c8c0b5feb8ee30a9/stopwords-cs.txt (MIT license) pub const CZECH: &[&str] = &[ "a", "aby", @@ -3075,4 +3075,3 @@ pub const SWEDISH: &[&str] = &[ "era", "vilkas", ]; -