feat: Make typo factor configurable

Fixes https://github.com/valeriansaliou/sonic/issues/394.

CONFIGURATION.md

@@ -98,6 +98,7 @@ Under `[search]`:
 * `suggest_limit_maximum` (type: _integer_, allowed: numbers, default: `20`) — Maximum suggested words limit for a suggest command (if the LIMIT command modifier is being used when issuing a SUGGEST command)
 * `list_limit_default` (type: _integer_, allowed: numbers, default: `100`) — Default listed words limit for a list command (if the LIMIT command modifier is not used when issuing a LIST command)
 * `list_limit_maximum` (type: _integer_, allowed: numbers, default: `500`) — Maximum listed words limit for a list command (if the LIMIT command modifier is being used when issuing a LIST command)
+* `typo_factor_word_lengths` (type: _array_, allowed: word lengths in bytes, default: `[4, 7, 10]`) — Word lengths from which one more typo is corrected in query and suggest commands (the default allows no typo up to 3 letters, 1 typo from 4 letters, 2 typos from 7 letters and 3 typos from 10 letters; typo correction can be slow and yield unexpected results, remove values to allow fewer typos or use `[]` to disable it)
 
 ### KV store configuration
 

config.cfg

@@ -55,6 +55,8 @@ suggest_limit_maximum = 20
 list_limit_default = 100
 list_limit_maximum = 500
 
+typo_factor_word_lengths = [4, 7, 10]
+
 
 [store.kv]
 path = "./data/store/kv/"

core/src/config.rs

@@ -120,6 +120,11 @@ pub struct SearchConfig {
     pub list_limit_default: u16,
 
     pub list_limit_maximum: u16,
+
+    /// Word lengths from which one more typo is allowed when correcting words
+    /// (e.g. `[4, 7, 10]` allows 1 typo from 4 letters, 2 typos from 7 letters
+    /// and 3 typos from 10 letters). Empty means no typo correction.
+    pub typo_factor_word_lengths: Vec<usize>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -386,6 +391,7 @@ pub(crate) mod tests {
         suggest_limit_maximum = 20
         list_limit_default = 100
         list_limit_maximum = 500
+        typo_factor_word_lengths = [4, 7, 10]
 
         [store.kv]
         path = "./data/store/kv/"

core/src/executor/search.rs

@@ -209,7 +209,10 @@ impl super::Executor {
                     continue 'terms;
                 }
 
-                let max_typo_factor = typo_factor(original_word_len);
+                let max_typo_factor = typo_factor(
+                    original_word_len,
+                    &self.app_conf.search.typo_factor_word_lengths,
+                );
                 let mut typo_factor = 1u32;
 
                 // TODO: Rework the Levenshtein query feature to avoid repeating

core/src/executor/suggest.rs

@@ -8,6 +8,7 @@
 use super::types::QuerySearchLimit;
 use crate::lexer::preprocessor::PreprocessorOutput;
 use crate::store::StoreItemPart;
+use crate::store::fst::typo_factor;
 
 impl super::Executor {
     pub fn suggest(
@@ -27,10 +28,12 @@ impl super::Executor {
             if let (Some(token), None) = (tokens.next(), tokens.next()) {
                 let len = token.as_original().len();
                 let term = token.into_normalized();
+                let max_typo_factor =
+                    typo_factor(len, &self.app_conf.search.typo_factor_word_lengths);
 
                 tracing::debug!("running suggest on word: {term:?}");
 
-                return match fst_store.suggest_words(term, len, limit as usize, None) {
+                return match fst_store.suggest_words(term, len, limit as usize, max_typo_factor) {
                     Some(words) => Ok(Some(words.map(|(k, _)| k))),
                     None => Ok(None),
                 };

core/src/store/fst/mod.rs

@@ -245,7 +245,7 @@ impl FstStore {
         // on user input.
         original_word_len: usize,
         limit: usize,
-        max_typo_factor: Option<u32>,
+        max_typo_factor: u32,
     ) -> Option<impl ExactSizeIterator<Item = (String, u16)> + DoubleEndedIterator + use<>> {
         use indexmap::IndexMap;
 
@@ -276,8 +276,6 @@ impl FstStore {
 
         // Try to fuzzy-suggest other words? (e.g. correct typos)
         if self.action_config.fuzzy_matching_enabled && found_words.len() < limit {
-            // Allow more typos in word as the word gets longer, up to a maximum limit
-            let max_typo_factor = max_typo_factor.unwrap_or(typo_factor(original_word_len));
             let mut typo_factor = 1u32;
 
             // TODO: Rework the Levenshtein query feature to avoid repeating
@@ -403,13 +401,14 @@ impl FstStore {
 }
 
 /// Allow more typos in word as the word gets longer, up to a maximum limit.
-pub(crate) fn typo_factor(word_len: usize) -> u32 {
-    match word_len {
-        1..=3 => 0,
-        4..=6 => 1,
-        7..=9 => 2,
-        _ => 3,
-    }
+///
+/// `typo_factor_word_lengths` lists the word lengths from which one more typo
+/// is allowed (see `search.typo_factor_word_lengths`).
+pub(crate) fn typo_factor(word_len: usize, typo_factor_word_lengths: &[usize]) -> u32 {
+    typo_factor_word_lengths
+        .iter()
+        .filter(|&&min_word_len| word_len >= min_word_len)
+        .count() as u32
 }
 
 // MARK: - Helpers

core/tests/common/config.rs

@@ -59,6 +59,7 @@ pub fn defaults_toml() -> String {
         suggest_limit_maximum = 20
         list_limit_default = 100
         list_limit_maximum = 500
+        typo_factor_word_lengths = [4, 7, 10]
 
         [store.kv]
         path = {kv_store_path:?}

core/tests/common/mod.rs

@@ -126,6 +126,31 @@ macro_rules! exec {
             .expect("QUERY should succeed")
     }};
 
+    ($executor:ident -> SUGGEST $collection:tt $bucket:tt $word:tt $(LANG($lang:expr))?) => {{
+        #[rustfmt::skip]
+        $executor.log(format!(
+            "SUGGEST {:?} {:?} {:?}{}",
+            $collection, $bucket, $word, exec!(internal_ lang_txt $($lang)?)
+        ));
+        let (c, b) = crate::common::bucket_ref!($collection, $bucket);
+        let preprocessor = sonic::lexer::preprocessor::Preprocessor::new(
+            $executor.app_conf.tokenization,
+            $executor.app_conf.normalization,
+            $executor.app_conf.stopwords.clone(),
+            false,
+            false,
+        );
+        $executor
+            .suggest(
+                c, b,
+                preprocessor.preprocess($word, exec!(internal_ lang $($lang)?)),
+                $executor.app_conf.search.suggest_limit_default,
+            )
+            .expect("SUGGEST should succeed")
+            .map(|words| words.collect::<Vec<String>>())
+            .unwrap_or_default()
+    }};
+
     ($executor:ident -> LIST $collection:tt $bucket:tt $(LIMIT($limit:expr))?) => {{
         $executor.log(format!("LIST {:?} {:?}", $collection, $bucket));
         let (c, b) = crate::common::bucket_ref!($collection, $bucket);

core/tests/typo_correction.rs

@@ -9,7 +9,8 @@
 //! A “typo factor” is chosen based on the token’s length. Those tests ensure
 //! typo correction works, and ensure no regression in the quality of the
 //! results.
-//! At the moment of this writing, the mapping function is the following:
+//! The mapping is configured by `search.typo_factor_word_lengths`. With the
+//! default value (`[4, 7, 10]`), it is the following:
 //!
 //! ```
 //! let mut typo_factor = match word.len() {
@@ -65,6 +66,69 @@ fn test_search_allows_typos() {
     ] LANG("eng"));
 }
 
+/// The word length -> typo factor mapping can be configured.
+///
+/// See <https://github.com/valeriansaliou/sonic/issues/394>.
+#[test]
+fn test_config_typo_factor_word_lengths() {
+    init_logging();
+    let executor = make_test_executor(|app_conf| {
+        // 1 typo from 3 letters, 2 typos from 5 letters, never 3 typos.
+        app_conf.search.typo_factor_word_lengths = vec![3, 5];
+    });
+
+    exec!(executor -> PUSH "messages" "user:1" "chat:1" ASTRONOMY_WORDS LANG("eng"));
+    exec!(executor -> TRIGGER consolidate);
+
+    #[rustfmt::skip]
+    let examples: [(&str, &[&str]); 3] = [
+        // 3-letter word, distance = 1.
+        ("sum", &["sun"]),
+        // 6-letter word, distance = 2.
+        ("nzbala", &["nebula"]),
+        // 10-letter word, distance = 3.
+        ("sapcecrzft", &[]),
+    ];
+
+    for (needle, expected_suggestions) in examples {
+        let expected_response: &[&str] = if expected_suggestions.is_empty() {
+            &[]
+        } else {
+            &["chat:1"]
+        };
+
+        let response = exec!(executor -> QUERY "messages" "user:1" needle LANG("eng"));
+        assert_eq!(response, expected_response, "QUERY {needle:?}");
+
+        let suggestions = exec!(executor -> SUGGEST "messages" "user:1" needle LANG("eng"));
+        assert_eq!(suggestions, expected_suggestions, "SUGGEST {needle:?}");
+    }
+}
+
+/// Typo correction can be disabled by configuring no word length.
+///
+/// See <https://github.com/valeriansaliou/sonic/issues/394>.
+#[test]
+fn test_config_typo_correction_disabled() {
+    init_logging();
+    let executor = make_test_executor(|app_conf| {
+        app_conf.search.typo_factor_word_lengths = vec![];
+    });
+
+    exec!(executor -> PUSH "messages" "user:1" "chat:1" ASTRONOMY_WORDS LANG("eng"));
+    exec!(executor -> TRIGGER consolidate);
+
+    // NOTE: All of those match with the default configuration
+    //   (see `test_search_allows_typos`).
+    for needle in ["ssun", "nzbula", "plusars", "sapcecrzft"] {
+        let response = exec!(executor -> QUERY "messages" "user:1" needle LANG("eng"));
+        assert_eq!(response, [] as [&str; 0], "QUERY {needle:?}");
+
+        let suggestions = exec!(executor -> SUGGEST "messages" "user:1" needle LANG("eng"));
+        assert_eq!(suggestions, [] as [&str; 0], "SUGGEST {needle:?}");
+    }
+}
+
 /// Ensures the order of words in search queries is insignificant.
 #[test]
 #[ignore = "Not supported yet (FIXME)"]

docs/search-results-ranking.md

@@ -72,6 +72,8 @@ fn query(query, app_config) -> [doc] {
             if update_score(results_matrix, doc, score, index, len(normalized(query))):
               alternates_try -= 1
 
+  // Thresholds are configured by `search.typo_factor_word_lengths`
+  // (shown with the default value: `[4, 7, 10]`).
   fn typo_factor(word_len) -> u32 {
     if      word_len <= 3: return 0
     else if word_len <= 6: return 1

server/src/config.rs

@@ -44,6 +44,7 @@ pub fn defaults_toml() -> &'static str {
     suggest_limit_maximum = 20
     list_limit_default = 100
     list_limit_maximum = 500
+    typo_factor_word_lengths = [4, 7, 10]
 
     [store.kv]
     path = "./data/store/kv/"