From 9bdd4a2024c45c7715c7ec1dab59c9670714a8e9 Mon Sep 17 00:00:00 2001 From: dualfroz Date: Sat, 10 Oct 2026 12:42:09 +0200 Subject: [PATCH] feat: Make typo factor configurable Fixes https://github.com/valeriansaliou/sonic/issues/394. --- CONFIGURATION.md | 1 + config.cfg | 2 ++ core/src/config.rs | 6 ++++ core/src/executor/search.rs | 5 ++- core/src/executor/suggest.rs | 5 ++- core/src/store/fst/mod.rs | 19 +++++----- core/tests/common/config.rs | 1 + core/tests/common/mod.rs | 25 +++++++++++++ core/tests/typo_correction.rs | 66 +++++++++++++++++++++++++++++++++- docs/search-results-ranking.md | 2 ++ server/src/config.rs | 1 + 11 files changed, 120 insertions(+), 13 deletions(-) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 04d61d4..dd4bfba 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -98,6 +98,7 @@ Under `[search]`: * `suggest_limit_maximum` (type: _integer_, allowed: numbers, default: `20`) — Maximum suggested words limit for a suggest command (if the LIMIT command modifier is being used when issuing a SUGGEST command) * `list_limit_default` (type: _integer_, allowed: numbers, default: `100`) — Default listed words limit for a list command (if the LIMIT command modifier is not used when issuing a LIST command) * `list_limit_maximum` (type: _integer_, allowed: numbers, default: `500`) — Maximum listed words limit for a list command (if the LIMIT command modifier is being used when issuing a LIST command) +* `typo_factor_word_lengths` (type: _array_, allowed: word lengths in bytes, default: `[4, 7, 10]`) — Word lengths from which one more typo is corrected in query and suggest commands (the default allows no typo up to 3 letters, 1 typo from 4 letters, 2 typos from 7 letters and 3 typos from 10 letters; typo correction can be slow and yield unexpected results, remove values to allow fewer typos or use `[]` to disable it) ### KV store configuration diff --git a/config.cfg b/config.cfg index 419e971..250dfd2 100644 --- a/config.cfg +++ b/config.cfg @@ -55,6 +55,8 @@ suggest_limit_maximum = 20 list_limit_default = 100 list_limit_maximum = 500 +typo_factor_word_lengths = [4, 7, 10] + [store.kv] path = "./data/store/kv/" diff --git a/core/src/config.rs b/core/src/config.rs index 07ba7ae..050adeb 100644 --- a/core/src/config.rs +++ b/core/src/config.rs @@ -120,6 +120,11 @@ pub struct SearchConfig { pub list_limit_default: u16, pub list_limit_maximum: u16, + + /// Word lengths from which one more typo is allowed when correcting words + /// (e.g. `[4, 7, 10]` allows 1 typo from 4 letters, 2 typos from 7 letters + /// and 3 typos from 10 letters). Empty means no typo correction. + pub typo_factor_word_lengths: Vec, } #[derive(Debug, Deserialize)] @@ -386,6 +391,7 @@ pub(crate) mod tests { suggest_limit_maximum = 20 list_limit_default = 100 list_limit_maximum = 500 + typo_factor_word_lengths = [4, 7, 10] [store.kv] path = "./data/store/kv/" diff --git a/core/src/executor/search.rs b/core/src/executor/search.rs index 07b2776..cac90d9 100644 --- a/core/src/executor/search.rs +++ b/core/src/executor/search.rs @@ -209,7 +209,10 @@ impl super::Executor { continue 'terms; } - let max_typo_factor = typo_factor(original_word_len); + let max_typo_factor = typo_factor( + original_word_len, + &self.app_conf.search.typo_factor_word_lengths, + ); let mut typo_factor = 1u32; // TODO: Rework the Levenshtein query feature to avoid repeating diff --git a/core/src/executor/suggest.rs b/core/src/executor/suggest.rs index 7694f68..3e8c2b1 100644 --- a/core/src/executor/suggest.rs +++ b/core/src/executor/suggest.rs @@ -8,6 +8,7 @@ use super::types::QuerySearchLimit; use crate::lexer::preprocessor::PreprocessorOutput; use crate::store::StoreItemPart; +use crate::store::fst::typo_factor; impl super::Executor { pub fn suggest( @@ -27,10 +28,12 @@ impl super::Executor { if let (Some(token), None) = (tokens.next(), tokens.next()) { let len = token.as_original().len(); let term = token.into_normalized(); + let max_typo_factor = + typo_factor(len, &self.app_conf.search.typo_factor_word_lengths); tracing::debug!("running suggest on word: {term:?}"); - return match fst_store.suggest_words(term, len, limit as usize, None) { + return match fst_store.suggest_words(term, len, limit as usize, max_typo_factor) { Some(words) => Ok(Some(words.map(|(k, _)| k))), None => Ok(None), }; diff --git a/core/src/store/fst/mod.rs b/core/src/store/fst/mod.rs index ed9eb5f..4de2826 100644 --- a/core/src/store/fst/mod.rs +++ b/core/src/store/fst/mod.rs @@ -245,7 +245,7 @@ impl FstStore { // on user input. original_word_len: usize, limit: usize, - max_typo_factor: Option, + max_typo_factor: u32, ) -> Option + DoubleEndedIterator + use<>> { use indexmap::IndexMap; @@ -276,8 +276,6 @@ impl FstStore { // Try to fuzzy-suggest other words? (e.g. correct typos) if self.action_config.fuzzy_matching_enabled && found_words.len() < limit { - // Allow more typos in word as the word gets longer, up to a maximum limit - let max_typo_factor = max_typo_factor.unwrap_or(typo_factor(original_word_len)); let mut typo_factor = 1u32; // TODO: Rework the Levenshtein query feature to avoid repeating @@ -403,13 +401,14 @@ impl FstStore { } /// Allow more typos in word as the word gets longer, up to a maximum limit. -pub(crate) fn typo_factor(word_len: usize) -> u32 { - match word_len { - 1..=3 => 0, - 4..=6 => 1, - 7..=9 => 2, - _ => 3, - } +/// +/// `typo_factor_word_lengths` lists the word lengths from which one more typo +/// is allowed (see `search.typo_factor_word_lengths`). +pub(crate) fn typo_factor(word_len: usize, typo_factor_word_lengths: &[usize]) -> u32 { + typo_factor_word_lengths + .iter() + .filter(|&&min_word_len| word_len >= min_word_len) + .count() as u32 } // MARK: - Helpers diff --git a/core/tests/common/config.rs b/core/tests/common/config.rs index 5087a96..acc3ed3 100644 --- a/core/tests/common/config.rs +++ b/core/tests/common/config.rs @@ -59,6 +59,7 @@ pub fn defaults_toml() -> String { suggest_limit_maximum = 20 list_limit_default = 100 list_limit_maximum = 500 + typo_factor_word_lengths = [4, 7, 10] [store.kv] path = {kv_store_path:?} diff --git a/core/tests/common/mod.rs b/core/tests/common/mod.rs index b9999f3..4010c4b 100644 --- a/core/tests/common/mod.rs +++ b/core/tests/common/mod.rs @@ -126,6 +126,31 @@ macro_rules! exec { .expect("QUERY should succeed") }}; + ($executor:ident -> SUGGEST $collection:tt $bucket:tt $word:tt $(LANG($lang:expr))?) => {{ + #[rustfmt::skip] + $executor.log(format!( + "SUGGEST {:?} {:?} {:?}{}", + $collection, $bucket, $word, exec!(internal_ lang_txt $($lang)?) + )); + let (c, b) = crate::common::bucket_ref!($collection, $bucket); + let preprocessor = sonic::lexer::preprocessor::Preprocessor::new( + $executor.app_conf.tokenization, + $executor.app_conf.normalization, + $executor.app_conf.stopwords.clone(), + false, + false, + ); + $executor + .suggest( + c, b, + preprocessor.preprocess($word, exec!(internal_ lang $($lang)?)), + $executor.app_conf.search.suggest_limit_default, + ) + .expect("SUGGEST should succeed") + .map(|words| words.collect::>()) + .unwrap_or_default() + }}; + ($executor:ident -> LIST $collection:tt $bucket:tt $(LIMIT($limit:expr))?) => {{ $executor.log(format!("LIST {:?} {:?}", $collection, $bucket)); let (c, b) = crate::common::bucket_ref!($collection, $bucket); diff --git a/core/tests/typo_correction.rs b/core/tests/typo_correction.rs index 62e4f03..df05755 100644 --- a/core/tests/typo_correction.rs +++ b/core/tests/typo_correction.rs @@ -9,7 +9,8 @@ //! A “typo factor” is chosen based on the token’s length. Those tests ensure //! typo correction works, and ensure no regression in the quality of the //! results. -//! At the moment of this writing, the mapping function is the following: +//! The mapping is configured by `search.typo_factor_word_lengths`. With the +//! default value (`[4, 7, 10]`), it is the following: //! //! ``` //! let mut typo_factor = match word.len() { @@ -65,6 +66,69 @@ fn test_search_allows_typos() { ] LANG("eng")); } +/// The word length -> typo factor mapping can be configured. +/// +/// See . +#[test] +fn test_config_typo_factor_word_lengths() { + init_logging(); + let executor = make_test_executor(|app_conf| { + // 1 typo from 3 letters, 2 typos from 5 letters, never 3 typos. + app_conf.search.typo_factor_word_lengths = vec![3, 5]; + }); + + exec!(executor -> PUSH "messages" "user:1" "chat:1" ASTRONOMY_WORDS LANG("eng")); + exec!(executor -> TRIGGER consolidate); + + #[rustfmt::skip] + let examples: [(&str, &[&str]); 3] = [ + // 3-letter word, distance = 1. + ("sum", &["sun"]), + // 6-letter word, distance = 2. + ("nzbala", &["nebula"]), + // 10-letter word, distance = 3. + ("sapcecrzft", &[]), + ]; + + for (needle, expected_suggestions) in examples { + let expected_response: &[&str] = if expected_suggestions.is_empty() { + &[] + } else { + &["chat:1"] + }; + + let response = exec!(executor -> QUERY "messages" "user:1" needle LANG("eng")); + assert_eq!(response, expected_response, "QUERY {needle:?}"); + + let suggestions = exec!(executor -> SUGGEST "messages" "user:1" needle LANG("eng")); + assert_eq!(suggestions, expected_suggestions, "SUGGEST {needle:?}"); + } +} + +/// Typo correction can be disabled by configuring no word length. +/// +/// See . +#[test] +fn test_config_typo_correction_disabled() { + init_logging(); + let executor = make_test_executor(|app_conf| { + app_conf.search.typo_factor_word_lengths = vec![]; + }); + + exec!(executor -> PUSH "messages" "user:1" "chat:1" ASTRONOMY_WORDS LANG("eng")); + exec!(executor -> TRIGGER consolidate); + + // NOTE: All of those match with the default configuration + // (see `test_search_allows_typos`). + for needle in ["ssun", "nzbula", "plusars", "sapcecrzft"] { + let response = exec!(executor -> QUERY "messages" "user:1" needle LANG("eng")); + assert_eq!(response, [] as [&str; 0], "QUERY {needle:?}"); + + let suggestions = exec!(executor -> SUGGEST "messages" "user:1" needle LANG("eng")); + assert_eq!(suggestions, [] as [&str; 0], "SUGGEST {needle:?}"); + } +} + /// Ensures the order of words in search queries is insignificant. #[test] #[ignore = "Not supported yet (FIXME)"] diff --git a/docs/search-results-ranking.md b/docs/search-results-ranking.md index bfc4bea..ac19091 100644 --- a/docs/search-results-ranking.md +++ b/docs/search-results-ranking.md @@ -72,6 +72,8 @@ fn query(query, app_config) -> [doc] { if update_score(results_matrix, doc, score, index, len(normalized(query))): alternates_try -= 1 + // Thresholds are configured by `search.typo_factor_word_lengths` + // (shown with the default value: `[4, 7, 10]`). fn typo_factor(word_len) -> u32 { if word_len <= 3: return 0 else if word_len <= 6: return 1 diff --git a/server/src/config.rs b/server/src/config.rs index 491d2d2..84c4505 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -44,6 +44,7 @@ pub fn defaults_toml() -> &'static str { suggest_limit_maximum = 20 list_limit_default = 100 list_limit_maximum = 500 + typo_factor_word_lengths = [4, 7, 10] [store.kv] path = "./data/store/kv/"