Upstream valeriansaliou/sonic state of the touched files before abd3d56935

CONFIGURATION.md

@@ -18,7 +18,7 @@ read from `SONIC_FOO__BAR__BAZ`.
 
 Under `[server]`:
 
-* `log_level` (type: _string_, allowed: `debug`, `info`, `warn`, `error`, default: `error`) — Verbosity of logging, set it to `error` in production
+* `log_level` (type: _string_, allowed: `debug`, `info`, `warn`, `error`, default: `warn`) — Verbosity of logging, set it to `error` in production
 
 ### Channel configuration
 

config.cfg

@@ -0,0 +1,81 @@
+# Sonic
+# Fast, lightweight and schema-less search backend
+# Configuration file
+# Example: https://github.com/valeriansaliou/sonic/blob/master/config.cfg
+
+[server]
+log_level = "warn"
+
+
+[channel]
+inet = "[::1]:1491"
+tcp_timeout = 300
+
+# NOTE: You can pass this via `SONIC_CHANNEL__AUTH_PASSWORD`.
+auth_password = "SecretPassword"
+
+
+# WARN: After making changes to normalization steps, you must rebuild Sonic’s
+#   index by re-ingesting all data. To avoid such breaking change, new features
+#   are disabled by default (opt-in). On major releases, some steps might
+#   become enabled by default (opt-out). Sonic won’t enable unreasonable
+#   defaults, but override if you need consistency.
+[normalization]
+# TODO(major): Remove explicit override.
+unicode_normalization = "none"
+# TODO(major): Remove explicit override.
+diacritic_folding_enabled = false
+# WARN: Enabling stemming greatly affects the quality of Sonic results.
+#   Enable only if you have a good reason to.
+stemming_enabled = false
+
+
+# WARN: After making changes to tokenization steps, you must rebuild Sonic’s
+#   index by re-ingesting all data. To avoid such breaking change, new features
+#   are disabled by default (opt-in). On major releases, some steps might
+#   become enabled by default (opt-out). Sonic won’t enable unreasonable
+#   defaults, but override if you need consistency.
+[tokenization]
+# TODO(major): Remove explicit override.
+detect_special_patterns = true
+# TODO(major): Remove explicit override.
+compat_split_special_patterns = true
+
+
+[search]
+query_limit_default = 10
+query_limit_maximum = 100
+query_alternates_try = 4
+query_minimum_term_idf_default = 0.1
+query_minimum_term_idf_minimum_object_count = 100
+
+suggest_limit_default = 5
+suggest_limit_maximum = 20
+
+list_limit_default = 100
+list_limit_maximum = 500
+
+
+[store.kv]
+path = "./data/store/kv/"
+
+retain_word_objects = 1000
+
+pool.inactive_after = 1800
+
+database.flush_after = 900
+database.compression_type = "zstd"
+database.parallelism = 2
+database.max_files = 100
+database.write_buffer_size = 16384
+database.write_ahead_log = true
+
+
+[store.fst]
+path = "./data/store/fst/"
+
+pool.inactive_after = 300
+
+graph.consolidate_after = 180
+graph.max_size = 2048
+graph.max_words = 250000

core/src/config.rs

@@ -0,0 +1,407 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+//! Sonic library configuration.
+//!
+//! It does not include server nor channel configuration, which are specific
+//! to the `sonic-server` binary.
+
+use std::collections::HashSet;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use serde::Deserialize;
+
+use crate::util::serde::env_var;
+
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    pub normalization: NormalizationConfig,
+
+    pub tokenization: TokenizationConfig,
+
+    pub stopwords: StopwordsConfig,
+
+    pub search: SearchConfig,
+
+    pub store: StoreConfig,
+}
+
+impl Config {
+    pub fn validate(&self) {
+        // Check 'write_buffer' for KV
+        if self.store.kv.database.write_buffer_size == Some(0) {
+            panic!("write_buffer for kv must not be zero");
+        }
+
+        // Check 'flush_after' for KV
+        if self.store.kv.database.flush_after >= self.store.kv.pool.inactive_after {
+            panic!("flush_after for kv must be strictly lower than inactive_after");
+        }
+
+        // Check 'consolidate_after' for FST
+        if self.store.fst.graph.consolidate_after >= self.store.fst.pool.inactive_after {
+            panic!("consolidate_after for fst must be strictly lower than inactive_after");
+        }
+    }
+}
+
+/// Configuration group for normalization options (Unicode normalization,
+/// stemming, lemmatization…).
+#[derive(Debug, Deserialize, Clone, Copy)]
+pub struct NormalizationConfig {
+    #[serde(with = "crate::util::serde::none_string_as_none")]
+    pub unicode_normalization: Option<UnicodeNormalization>,
+
+    pub diacritic_folding_enabled: bool,
+
+    #[cfg(feature = "stemming")]
+    pub stemming_enabled: bool,
+}
+
+#[derive(Deserialize, Debug, Clone, Copy)]
+#[serde(rename_all = "snake_case")]
+pub enum UnicodeNormalization {
+    /// Unicode Normalization Form C.
+    Nfc,
+    /// Unicode Normalization Form KC.
+    Nfkc,
+}
+
+/// Configuration group for tokenization options.
+#[derive(Debug, Deserialize, Clone, Copy)]
+pub struct TokenizationConfig {
+    pub detect_special_patterns: bool,
+
+    #[serde(alias = "split_special_patterns")]
+    pub compat_split_special_patterns: bool,
+}
+
+#[derive(Debug, Deserialize, Clone, Default)]
+pub struct StopwordsConfig {
+    #[serde(deserialize_with = "to_stopwords")]
+    pub allow: HashSet<String>,
+
+    #[serde(deserialize_with = "to_stopwords")]
+    pub deny: HashSet<String>,
+}
+
+fn to_stopwords<'de, D>(deserializer: D) -> Result<HashSet<String>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    use unicode_normalization::UnicodeNormalization as _;
+
+    let vec: Vec<Box<str>> = Deserialize::deserialize(deserializer)?;
+    let stopwords_iter = vec.into_iter().map(|s| s.nfkd().to_string());
+    Ok(HashSet::from_iter(stopwords_iter))
+}
+
+#[derive(Debug, Deserialize)]
+pub struct SearchConfig {
+    pub query_limit_default: u16,
+
+    pub query_limit_maximum: u16,
+
+    pub query_alternates_try: usize,
+
+    pub query_minimum_term_idf_default: f32,
+
+    pub query_minimum_term_idf_minimum_object_count: u64,
+
+    pub suggest_limit_default: u16,
+
+    pub suggest_limit_maximum: u16,
+
+    pub list_limit_default: u16,
+
+    pub list_limit_maximum: u16,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct StoreConfig {
+    pub kv: Arc<KvStoreConfig>,
+
+    pub fst: Arc<FstStoreConfig>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct KvStoreConfig {
+    #[serde(deserialize_with = "env_var::path_buf")]
+    pub path: PathBuf,
+
+    pub retain_word_objects: usize,
+
+    pub pool: KvStorePoolConfig,
+
+    pub database: KvStoreDatabaseConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct KvStorePoolConfig {
+    pub inactive_after: u64,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct KvStoreDatabaseConfig {
+    pub flush_after: u64,
+
+    pub write_ahead_log: bool,
+
+    /// Whether or not to compress.
+    ///
+    /// Will get overriden if [`compression_type`](Self::compression_type) is
+    /// also specified.
+    #[serde(default)]
+    pub compress: Option<bool>,
+
+    #[serde(default)]
+    pub parallelism: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "max_files")]
+    pub max_open_files: Option<i32>,
+
+    // TODO(major): Make this MB, as in Kvrocks.
+    /// WARN: In KB!
+    #[serde(default = "default_write_buffer_size")]
+    #[serde(alias = "write_buffer")]
+    pub write_buffer_size: Option<usize>,
+
+    #[serde(default)]
+    pub max_write_buffer_number: Option<i32>,
+
+    #[serde(default)]
+    pub min_write_buffer_number: Option<i32>,
+
+    #[serde(default)]
+    pub min_write_buffer_number_to_merge: Option<i32>,
+
+    #[serde(default)]
+    pub block_cache_size: Option<u32>,
+
+    #[serde(default)]
+    pub cache_index_and_filter_blocks: Option<bool>,
+
+    #[serde(default)]
+    #[serde(alias = "compression")]
+    #[serde(deserialize_with = "to_rocksdb_compression_type_opt")]
+    pub compression_type: Option<rocksdb::DBCompressionType>,
+
+    #[serde(default)]
+    #[serde(alias = "wal_compression")]
+    #[serde(deserialize_with = "to_rocksdb_compression_type_opt")]
+    pub wal_compression_type: Option<rocksdb::DBCompressionType>,
+
+    #[serde(default)]
+    pub wal_ttl_seconds: Option<u64>,
+
+    #[serde(default)]
+    pub wal_size_limit_mb: Option<u64>,
+
+    #[serde(default)]
+    pub wal_bytes_per_sync: Option<u64>,
+
+    #[serde(default)]
+    #[serde(deserialize_with = "to_rocksdb_recovery_mode_opt")]
+    pub wal_recovery_mode: Option<rocksdb::DBRecoveryMode>,
+
+    #[serde(default)]
+    pub compression_level: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "compression_start_level")]
+    pub min_level_to_compress: Option<std::ffi::c_int>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_file_num_compaction_trigger")]
+    pub level_zero_file_num_compaction_trigger: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_slowdown_writes_trigger")]
+    pub level_zero_slowdown_writes_trigger: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_stop_writes_trigger")]
+    pub level_zero_stop_writes_trigger: Option<i32>,
+
+    #[serde(default)]
+    pub max_bytes_for_level_base: Option<u64>,
+
+    #[serde(default)]
+    pub max_bytes_for_level_multiplier: Option<f64>,
+
+    #[serde(default)]
+    pub target_file_size_base: Option<u64>,
+
+    #[serde(default)]
+    pub max_background_jobs: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "max_compactions")]
+    pub max_subcompactions: Option<u32>,
+
+    #[serde(default)]
+    pub max_flushes: Option<i32>,
+
+    #[serde(default)]
+    pub stats_dump_period_sec: Option<u32>,
+}
+
+fn default_write_buffer_size() -> Option<usize> {
+    Some(16384)
+}
+
+fn parse_rocksdb_compression_type<E: serde::de::Error>(
+    str: &str,
+) -> Result<rocksdb::DBCompressionType, E> {
+    // NOTE: Some values are not available because not compiled in rocksdb
+    //   (feature flag is off).
+    match str.to_ascii_lowercase().as_str() {
+        "none" => Ok(rocksdb::DBCompressionType::None),
+        // "snappy" => Ok(rocksdb::DBCompressionType::Snappy),
+        // "zlib" => Ok(rocksdb::DBCompressionType::Zlib),
+        // "bz2" => Ok(rocksdb::DBCompressionType::Bz2),
+        // "lz4" => Ok(rocksdb::DBCompressionType::Lz4),
+        // "lz4hc" => Ok(rocksdb::DBCompressionType::Lz4hc),
+        "zstd" => Ok(rocksdb::DBCompressionType::Zstd),
+        _ => Err(serde::de::Error::unknown_variant(
+            str,
+            &[
+                "none",
+                // "snappy",
+                // "zlib",
+                // "bz2",
+                // "lz4",
+                // "lz4hc",
+                "zstd",
+            ],
+        )),
+    }
+}
+
+fn to_rocksdb_compression_type_opt<'de, D>(
+    deserializer: D,
+) -> Result<Option<rocksdb::DBCompressionType>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    let str: Option<String> = Deserialize::deserialize(deserializer)?;
+    str.map(|s| parse_rocksdb_compression_type(&s)).transpose()
+}
+
+fn parse_rocksdb_recovery_mode<E: serde::de::Error>(
+    str: &str,
+) -> Result<rocksdb::DBRecoveryMode, E> {
+    match str.to_ascii_lowercase().as_str() {
+        "tolerate_corrupted_tail_records" | "TolerateCorruptedTailRecords" => {
+            Ok(rocksdb::DBRecoveryMode::TolerateCorruptedTailRecords)
+        }
+        "absolute_consistency" | "AbsoluteConsistency" => {
+            Ok(rocksdb::DBRecoveryMode::AbsoluteConsistency)
+        }
+        "point_in_time" | "PointInTime" => Ok(rocksdb::DBRecoveryMode::PointInTime),
+        "skip_any_corrupted_record" | "SkipAnyCorruptedRecord" => {
+            Ok(rocksdb::DBRecoveryMode::SkipAnyCorruptedRecord)
+        }
+        _ => Err(serde::de::Error::unknown_variant(
+            str,
+            &[
+                "tolerate_corrupted_tail_records",
+                "absolute_consistency",
+                "point_in_time",
+                "skip_any_corrupted_record",
+            ],
+        )),
+    }
+}
+
+fn to_rocksdb_recovery_mode_opt<'de, D>(
+    deserializer: D,
+) -> Result<Option<rocksdb::DBRecoveryMode>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    let str: Option<String> = Deserialize::deserialize(deserializer)?;
+    str.map(|s| parse_rocksdb_recovery_mode(&s)).transpose()
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStoreConfig {
+    #[serde(deserialize_with = "env_var::path_buf")]
+    pub path: PathBuf,
+
+    pub pool: FstStorePoolConfig,
+
+    pub graph: FstStoreGraphConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStorePoolConfig {
+    pub inactive_after: u64,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStoreGraphConfig {
+    pub consolidate_after: u64,
+
+    pub max_size: usize,
+
+    pub max_words: usize,
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    pub(crate) fn defaults_toml() -> &'static str {
+        r#"
+        [channel]
+        inet = "[::1]:1491"
+        tcp_timeout = 300
+
+        [normalization]
+        unicode_normalization = "none"
+        diacritic_folding_enabled = true
+        stemming_enabled = false
+
+        [tokenization]
+        detect_special_patterns = true
+        compat_split_special_patterns = false
+
+        [stopwords]
+        allow = []
+        deny = []
+
+        [search]
+        query_limit_default = 10
+        query_limit_maximum = 100
+        query_alternates_try = 4
+        query_minimum_term_idf_default = 0.1
+        query_minimum_term_idf_minimum_object_count = 100
+        suggest_limit_default = 5
+        suggest_limit_maximum = 20
+        list_limit_default = 100
+        list_limit_maximum = 500
+
+        [store.kv]
+        path = "./data/store/kv/"
+        retain_word_objects = 1000
+        pool.inactive_after = 1800
+        database.flush_after = 900
+        database.compression_type = "zstd"
+        database.parallelism = 2
+        database.write_ahead_log = true
+
+        [store.fst]
+        path = "./data/store/fst/"
+        pool.inactive_after = 300
+        graph.consolidate_after = 180
+        graph.max_size = 2048
+        graph.max_words = 250000
+        "#
+    }
+}

core/src/executor/search.rs

@@ -0,0 +1,613 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+use indexmap::IndexMap;
+
+use super::types::{QueryMatchScore, QueryResultScore, QuerySearchLimit, QuerySearchOffset};
+use crate::lexer::itertools::UniqueBy;
+use crate::lexer::preprocessor::{PreprocessorOutput, Token};
+use crate::store::StoreItemPart;
+use crate::store::fst::typo_factor;
+use crate::store::kv::KvStoreActionReadOnly;
+use crate::store::{StoreObjectIid, StoreTermHash};
+use crate::util::hash::NoopU32HasherBuilder;
+
+impl super::Executor {
+    pub fn search(
+        &self,
+        collection: StoreItemPart,
+        bucket: StoreItemPart,
+        input: PreprocessorOutput,
+        limit: QuerySearchLimit,
+        offset: QuerySearchOffset,
+    ) -> Result<Vec<String>, ()> {
+        // Important: acquire database access read lock, and reference it in context. This \
+        //   prevents the database from being erased while using it in this block.
+        let _kv_read_guard = self.kv_pool.lock_read_access();
+        let _fst_read_guard = self.fst_pool.lock_read_access();
+
+        let (Ok(kv_store), Ok(fst_store)) = (
+            self.kv_pool.acquire(false, collection, None, |_| {}),
+            self.fst_pool.acquire(collection, bucket),
+        ) else {
+            return Err(());
+        };
+
+        let Some(kv_store) = kv_store else {
+            tracing::debug!(
+                "collection store does not exist, consider {bucket:?} from {collection:?} empty"
+            );
+            return Ok(vec![]);
+        };
+
+        let (higher_limit, mut alternates_try) = (
+            self.app_conf.store.kv.retain_word_objects,
+            self.app_conf.search.query_alternates_try,
+        );
+
+        let (mut minimum_idf, idf_min_doc_count) = (
+            self.app_conf.search.query_minimum_term_idf_default,
+            (self.app_conf.search).query_minimum_term_idf_minimum_object_count,
+        );
+
+        let (prefix_matching_enabled, fuzzy_matching_enabled) = (
+            self.fst_pool.fst_action_config.prefix_matching_enabled,
+            self.fst_pool.fst_action_config.fuzzy_matching_enabled,
+        );
+
+        // Important: acquire bucket store read lock
+        executor_kv_lock_read!(kv_store);
+
+        let kv_action = kv_store.access_read_only(bucket);
+
+        let document_count = kv_action
+            .get_object_count()
+            .map_err(|error| tracing::warn!("{error:?}"))? as u64;
+
+        if document_count < idf_min_doc_count {
+            tracing::debug!(
+                "ignoring minimum_term_idf ({minimum_idf}) as document_count is too low ({document_count}<{idf_min_doc_count})"
+            );
+            minimum_idf = 0.;
+        }
+
+        // Collect all terms so we know the count right ahead.
+        // PERF: This helps allocating the correct amounts of memory.
+        let tokens: Vec<Token> =
+            UniqueBy::new_with_hasher(input.tokens(), Token::hash, NoopU32HasherBuilder).collect();
+        let term_count = tokens.len();
+
+        // Store scores for each found IID. Results will then be sorted by
+        // score before being returned. Scores are basically the sum of
+        // Levenshtein distances for each term in the query. Lower score
+        // means better result.
+        // NOTE: We use `IndexMap` instead of `HashMap` to preserve
+        //   insertion order, which correlates to reverse data ingestion
+        //   order.
+        // NOTE: `capacity = 24` to reduce initial grows.
+        let mut scoring_matrix: IndexMap<StoreObjectIid, Vec<Option<QueryMatchScore>>> =
+            IndexMap::with_capacity(24usize.min(usize::from(limit)));
+
+        // Look for exact matches.
+        'matches: for (idx, token) in tokens.iter().enumerate() {
+            let term_hash = token.hash();
+            let term = token.as_normalized();
+
+            let mut iids = kv_action
+                .get_term_to_iids(term_hash)
+                .unwrap_or(None)
+                .unwrap_or_default();
+
+            // Look for exact matches normalized differently if the Sonic
+            // index isn’t normalized.
+            if !token.is_special() && self.app_conf.normalization.unicode_normalization.is_none() {
+                use unicode_normalization::UnicodeNormalization as _;
+
+                let mut nfc = kv_action
+                    .get_term_to_iids(StoreTermHash::from(term.nfc().to_string().as_str()))
+                    .unwrap_or(None)
+                    .unwrap_or_default();
+                iids.append(&mut nfc);
+
+                let mut nfd = kv_action
+                    .get_term_to_iids(StoreTermHash::from(term.nfd().to_string().as_str()))
+                    .unwrap_or(None)
+                    .unwrap_or_default();
+                iids.append(&mut nfd);
+            };
+
+            tracing::debug!("got exact search executor iids: {iids:?} for term: {term:?}");
+
+            let document_frequency = document_frequency(term_hash, &kv_action);
+
+            // Filter out minimum IDF.
+            // PERF: Filtering `minimum_idf > 0` to save some computation.
+            if minimum_idf > 0. {
+                let idf = (document_count as f32 / document_frequency as f32).ln();
+                if idf < minimum_idf {
+                    tracing::debug!(
+                        "skipping term {term:?} because idf too low ({idf}<{minimum_idf})"
+                    );
+                    continue;
+                }
+            }
+
+            let bm25_score = bm25_lite_idf(document_count, document_frequency);
+
+            for iid in iids.into_iter() {
+                // Assign a base score of `1` as those are exact matches.
+                let inserted =
+                    update_score(&mut scoring_matrix, iid, 1. * bm25_score, idx, term_count);
+
+                if inserted {
+                    // Higher limit now reached?
+                    // Stop acquiring new suggested IIDs now.
+                    if scoring_matrix.len() >= higher_limit {
+                        tracing::trace!(?term, "got enough completed results for term");
+
+                        break 'matches;
+                    }
+                }
+            }
+        }
+
+        #[cfg(debug_assertions)]
+        tracing::debug!(?scoring_matrix);
+
+        // Look for words containing `term` as prefix.
+        if scoring_matrix.len() < higher_limit && alternates_try > 0 && prefix_matching_enabled {
+            tracing::debug!(
+                "not enough iids were found ({}/{higher_limit}), looking for prefixes",
+                scoring_matrix.len(),
+            );
+
+            'terms: for (idx, token) in tokens.iter().enumerate() {
+                let original_len = token.as_original().len();
+                let term = token.as_normalized();
+
+                let Some(suggestions) = fst_store.lookup_begins(term, original_len) else {
+                    tracing::trace!("did not get any completed word for term {term:?}");
+                    continue 'terms;
+                };
+
+                merge_suggestions(
+                    suggestions.map(|(w, distance)| (w, prefix_score(distance, original_len))),
+                    &mut scoring_matrix,
+                    term,
+                    idx,
+                    term_count,
+                    &kv_action,
+                    &mut alternates_try,
+                    higher_limit,
+                    document_count,
+                    minimum_idf,
+                );
+            }
+        }
+
+        #[cfg(debug_assertions)]
+        tracing::debug!(?scoring_matrix);
+
+        // Look for words like `term` (fuzzy matching).
+        if scoring_matrix.len() < higher_limit && alternates_try > 0 && fuzzy_matching_enabled {
+            tracing::debug!(
+                "not enough iids were found ({}/{higher_limit}), looking for fuzzy matches",
+                scoring_matrix.len(),
+            );
+
+            'terms: for (idx, token) in tokens.iter().enumerate() {
+                let original_word_len = token.as_original().len();
+                let term = token.as_normalized();
+
+                // Skip term if it’s special (we’d want exact matches only).
+                if token.is_special() {
+                    tracing::debug!("skipping fuzzy search for {term:?}: term is special");
+                    continue 'terms;
+                }
+
+                let max_typo_factor = typo_factor(original_word_len);
+                let mut typo_factor = 1u32;
+
+                // TODO: Rework the Levenshtein query feature to avoid repeating
+                //   the same query over and over again. Maybe try to see if
+                //   `fst_levenshtein` can return distances in its response.
+                while alternates_try > 0 && typo_factor <= max_typo_factor {
+                    let Some(suggestions) = fst_store.lookup_typos(term, typo_factor) else {
+                        tracing::trace!("did not get any completed word for term {term:?}");
+                        continue 'terms;
+                    };
+
+                    merge_suggestions(
+                        suggestions
+                            .map(|(w, distance)| (w, typo_score(distance, original_word_len))),
+                        &mut scoring_matrix,
+                        term,
+                        idx,
+                        term_count,
+                        &kv_action,
+                        &mut alternates_try,
+                        higher_limit,
+                        document_count,
+                        minimum_idf,
+                    );
+
+                    typo_factor += 1;
+                }
+            }
+        }
+
+        #[cfg(debug_assertions)]
+        tracing::debug!(?scoring_matrix);
+
+        // Switch to implicit `AND` if query contains a special token.
+        // NOTE: When a user queries for a special token (e.g. UUID),
+        //   they expect only exact matches to be returned. If one term is
+        //   considered special, we drop all results missing at least one
+        //   term. It’s not the most efficient (compared to not storing the
+        //   result in the first place) but it’s an edge case and the cost
+        //   is negligible.
+        let one_term_is_special = tokens.iter().any(Token::is_special);
+        if one_term_is_special {
+            let mut to_remove = Vec::<StoreObjectIid>::new();
+
+            for (&iid, scores) in scoring_matrix.iter() {
+                if scores.iter().any(Option::is_none) {
+                    to_remove.push(iid);
+                }
+            }
+
+            for iid in to_remove {
+                scoring_matrix.swap_remove(&iid);
+            }
+        }
+
+        // Flatten scores, taking into account missing matches (thanks to
+        // `None`).
+        let found_iids = scoring_matrix
+            .into_iter()
+            .map(|(iid, scores)| (iid, overall_score(&scores)));
+
+        // Sort found IIDs.
+        let all_iids = {
+            let mut all_iids = found_iids.collect::<Vec<_>>();
+            all_iids.sort_by(|a, b| a.1.total_cmp(&b.1).reverse());
+            all_iids.into_iter().map(|(iid, _score)| iid)
+        };
+
+        // Resolve OIDs from IIDs
+        // Notice: we also proceed paging from there
+        let (limit_usize, offset_usize) = (limit as usize, offset as usize);
+        let mut result_oids = Vec::with_capacity(limit_usize);
+
+        'paging: for (index, found_iid) in all_iids.skip(offset_usize).enumerate() {
+            // Stop there?
+            if index >= limit_usize {
+                break 'paging;
+            }
+
+            // Read IID-to-OID for this found IID
+            if let Ok(Some(oid)) = kv_action.get_iid_to_oid(found_iid) {
+                result_oids.push(oid);
+            } else {
+                tracing::error!("failed getting search executor iid-to-oid");
+            }
+        }
+
+        tracing::info!("got search executor final oids: {:?}", result_oids);
+
+        Ok(result_oids)
+    }
+}
+
+/// Inversely proportional to `lev_distance / word_len`, decreasing slowly
+/// towards `f(20) = 0.5`. Will never reach `0`.
+fn prefix_score(lev_distance: u16, word_len: usize) -> f32 {
+    // NOTE: Will be `> 1` in practice.
+    let lev_ratio = lev_distance as f32 / word_len as f32;
+
+    // NOTE: `20` means that auto-completed words 20 times longer than the
+    //   original word get a score of `0.5`. It’s just a magic number, it has
+    //   no further meaning. It just feels ok.
+    20. / (20. + lev_ratio)
+}
+
+#[cfg(test)]
+#[test]
+fn test_prefix_score() {
+    // Auto-complete 2 times longer.
+    for n in [2, 4, 8] {
+        assert_eq!(prefix_score(n, n as usize), 0.95238096);
+    }
+
+    // Auto-complete 4 times longer.
+    for n in [2, 4, 8] {
+        assert_eq!(prefix_score(3 * n, n as usize), 0.8695652);
+    }
+
+    // Auto-complete 1 character.
+    assert_eq!(prefix_score(1, 3), 0.9836065);
+    assert_eq!(prefix_score(1, 4), 0.9876543);
+    assert_eq!(prefix_score(1, 5), 0.99009895);
+    assert_eq!(prefix_score(1, 6), 0.9917356);
+
+    // Auto-complete 2 characters.
+    assert_eq!(prefix_score(2, 3), 0.96774197);
+    assert_eq!(prefix_score(2, 4), 0.9756098);
+    assert_eq!(prefix_score(2, 5), 0.98039216);
+    assert_eq!(prefix_score(2, 6), 0.9836065);
+
+    // More auto-complete means lower score.
+    for n in [1, 2, 4, 8] {
+        for word_len in [2, 4, 8, 10] {
+            assert!(prefix_score(n + 1, word_len as usize) < prefix_score(n, word_len as usize));
+        }
+    }
+}
+
+/// Levenshtein distance proportional to word length.
+fn typo_score(lev_distance: u16, word_len: usize) -> f32 {
+    debug_assert!(
+        (lev_distance as usize) < word_len,
+        "{lev_distance} >= {word_len}"
+    );
+
+    // SAFETY: `.min(1)` isn’t strictly necessary as `lev_distance` should
+    //   always be `< word_len`, but it’s there as a safety precaution.
+    let lev_ratio = (lev_distance as f32 / word_len as f32).min(1.);
+
+    1. - lev_ratio
+}
+
+#[cfg(test)]
+#[test]
+fn test_typo_score() {
+    // No typo.
+    assert_eq!(typo_score(0, 1), 1.);
+    assert_eq!(typo_score(0, 2), 1.);
+
+    // 1 typo.
+    assert_eq!(typo_score(1, 3), 0.6666666);
+    assert_eq!(typo_score(1, 4), 0.75);
+    assert_eq!(typo_score(1, 5), 0.8);
+    // 1 typo always scores lower than 0.
+    for n in 2..=8 {
+        assert!(typo_score(1, n) < typo_score(0, n), "n={n}");
+    }
+
+    // 2 typos.
+    assert_eq!(typo_score(2, 5), 0.6);
+    assert_eq!(typo_score(2, 6), 0.6666666);
+    assert_eq!(typo_score(2, 7), 0.71428573);
+    // 2 typos always scores lower than 1.
+    for n in 3..=8 {
+        assert!(typo_score(2, n) < typo_score(1, n), "n={n}");
+    }
+
+    // A lot of typos (length has no impact, proportion has).
+    assert_eq!(typo_score(1 * 20, 2 * 20), typo_score(1, 2));
+    assert_eq!(typo_score(3 * 20, 7 * 20), typo_score(3, 7));
+}
+
+fn overall_score(scores: &[Option<QueryMatchScore>]) -> QueryResultScore {
+    let total = scores.iter().map(|opt| opt.unwrap_or(0f32)).sum::<f32>();
+    let count = scores.len() as f32;
+
+    #[allow(clippy::let_and_return)]
+    let average = total / count;
+
+    average
+}
+
+#[cfg(test)]
+#[test]
+fn test_overall_score() {
+    const MISSING: Option<QueryMatchScore> = None;
+    const EXACT_MATCH: Option<QueryMatchScore> = Some(1.);
+
+    // Max score for exact matches.
+    assert_eq!(overall_score(&[EXACT_MATCH; 1]), 1.);
+    assert_eq!(overall_score(&[EXACT_MATCH; 2]), 1.);
+    assert_eq!(overall_score(&[EXACT_MATCH; 3]), 1.);
+    assert_eq!(overall_score(&[EXACT_MATCH; 4]), 1.);
+
+    // Lowest score for missing matches.
+    assert_eq!(overall_score(&[MISSING; 1]), 0.);
+    assert_eq!(overall_score(&[MISSING; 2]), 0.);
+    assert_eq!(overall_score(&[MISSING; 3]), 0.);
+    assert_eq!(overall_score(&[MISSING; 4]), 0.);
+
+    // Auto-complete > fuzzy matching (not always, but in most cases).
+    assert!(overall_score(&[Some(prefix_score(5, 4))]) > overall_score(&[Some(typo_score(1, 10))]));
+
+    // Missing one term.
+    assert_eq!(overall_score(&[MISSING, EXACT_MATCH]), 1. / 2.);
+    assert_eq!(overall_score(&[MISSING, EXACT_MATCH, EXACT_MATCH]), 2. / 3.);
+    // Missing one term is better than missing all terms.
+    assert!(overall_score(&[MISSING, EXACT_MATCH]) > overall_score(&[MISSING]));
+
+    // Term order has no meaning.
+    assert_eq!(
+        overall_score(&[
+            EXACT_MATCH,
+            Some(prefix_score(2, 3)),
+            Some(typo_score(2, 7))
+        ]),
+        overall_score(&[
+            Some(typo_score(2, 7)),
+            Some(prefix_score(2, 3)),
+            EXACT_MATCH
+        ])
+    );
+
+    // All typos in one term is like the same total across multiple terms.
+    // NOTE: This is not a requirement, it’s just a non-regression test.
+    assert_eq!(
+        overall_score(&[Some(typo_score(1, 7)); 2]),
+        overall_score(&[Some(typo_score(2, 7)), EXACT_MATCH])
+    );
+    assert_eq!(
+        overall_score(&[Some(typo_score(1, 7)); 3]),
+        overall_score(&[Some(typo_score(3, 7)), EXACT_MATCH, EXACT_MATCH])
+    );
+
+    // Examples for “The brown fox jumps over the lazy dog”:
+    // “brown fox jumps”
+    assert_eq!(overall_score(&[EXACT_MATCH, EXACT_MATCH, EXACT_MATCH]), 1.);
+    // “brown fox jum”
+    assert_eq!(
+        overall_score(&[EXACT_MATCH, EXACT_MATCH, Some(prefix_score(2, 3))]),
+        0.9892473
+    );
+    // “bron fox jum”
+    assert_eq!(
+        overall_score(&[
+            Some(typo_score(1, 5)),
+            EXACT_MATCH,
+            Some(prefix_score(2, 3))
+        ]),
+        0.92258066
+    );
+    // “brown fox”
+    assert_eq!(overall_score(&[EXACT_MATCH, EXACT_MATCH,]), 1.);
+    // “brown fox eats”
+    assert_eq!(
+        overall_score(&[EXACT_MATCH, EXACT_MATCH, MISSING]),
+        0.6666667
+    ); // 2/3
+}
+
+fn document_frequency(term_hash: StoreTermHash, kv_action: &KvStoreActionReadOnly<'_>) -> u64 {
+    kv_action
+        .get_term_to_iids(term_hash)
+        .inspect_err(|err| tracing::error!("{err:?}"))
+        .unwrap_or(None)
+        .map_or(0, |iids| iids.len()) as u64
+}
+
+fn bm25_lite_idf(document_count: u64, document_frequency: u64) -> f32 {
+    debug_assert!(
+        document_frequency <= document_count,
+        "{document_frequency} > {document_count}"
+    );
+
+    let document_count = document_count.max(document_frequency) as f64;
+    let df = document_frequency as f64;
+
+    (1.0 + (document_count - df + 0.5) / (df + 0.5)).ln() as f32
+}
+
+#[allow(clippy::too_many_arguments)] // We’ll refactor this someday, and it’ not public anyway.
+fn merge_suggestions(
+    suggestions: impl Iterator<Item = (String, QueryMatchScore)>,
+    scoring_matrix: &mut IndexMap<StoreObjectIid, Vec<Option<QueryMatchScore>>>,
+    term: &str,
+    term_idx: usize,
+    term_count: usize,
+    kv_action: &KvStoreActionReadOnly<'_>,
+    alternates_try: &mut usize,
+    higher_limit: usize,
+    document_count: u64,
+    minimum_idf: f32,
+) {
+    'suggestions: for (suggested_word, base_score) in suggestions {
+        // Do not load base results twice for same term as base term
+        if suggested_word.eq(term) {
+            continue;
+        }
+
+        tracing::trace!(?term, ?suggested_word, "got completed word for term");
+
+        let suggested_term_hash = StoreTermHash::from(suggested_word.as_str());
+        let suggested_iids = match kv_action.get_term_to_iids(suggested_term_hash) {
+            Ok(Some(suggested_iids)) => suggested_iids,
+            Ok(None) => continue,
+            Err(_) => continue,
+        };
+
+        let document_frequency = document_frequency(suggested_term_hash, kv_action);
+
+        // Filter out minimum IDF.
+        // PERF: Filtering `minimum_idf > 0` to save some computation.
+        if minimum_idf > 0. {
+            let idf = (document_count as f32 / document_frequency as f32).ln();
+            if idf < minimum_idf {
+                tracing::debug!(
+                    "skipping term {suggested_word:?} because idf too low ({idf}<{minimum_idf})"
+                );
+                continue;
+            }
+        }
+
+        let bm25_score = bm25_lite_idf(document_count, document_frequency);
+
+        let suggestion_score = base_score * bm25_score;
+
+        for suggested_iid in suggested_iids.into_iter().take(*alternates_try) {
+            // SAFETY: We can reach at most `alternates_try`.
+            *alternates_try = unsafe { alternates_try.unchecked_sub(1) };
+
+            let inserted = update_score(
+                scoring_matrix,
+                suggested_iid,
+                suggestion_score,
+                term_idx,
+                term_count,
+            );
+
+            if inserted {
+                // Higher limit now reached?
+                // Stop acquiring new suggested IIDs now.
+                if scoring_matrix.len() >= higher_limit {
+                    tracing::trace!(?term, "got enough completed results for term");
+
+                    break 'suggestions;
+                }
+            }
+        }
+    }
+
+    tracing::trace!(
+        ?term,
+        "done completing results for term, now {} total results",
+        scoring_matrix.len()
+    );
+}
+
+fn update_score(
+    scoring_matrix: &mut IndexMap<StoreObjectIid, Vec<Option<QueryMatchScore>>>,
+    iid: StoreObjectIid,
+    score: QueryMatchScore,
+    term_idx: usize,
+    term_count: usize,
+) -> bool {
+    match scoring_matrix.entry(iid) {
+        // If entry already exists, use lowest score.
+        indexmap::map::Entry::Occupied(mut occupied_entry) => {
+            // SAFETY: We always initialize vecs with `term_count` entries.
+            let entry_score = unsafe { occupied_entry.get_mut().get_unchecked_mut(term_idx) };
+
+            let new_score = entry_score.map_or(score, |entry_score| score.min(entry_score));
+
+            tracing::trace!(entry_score, new_score, "Updating to min score");
+            *entry_score = Some(new_score);
+
+            false
+        }
+        // If entry does not exist, insert score.
+        indexmap::map::Entry::Vacant(vacant_entry) => {
+            let mut scores = vec![None; term_count];
+
+            tracing::trace!(new_score = score, "Inserting new score");
+            // SAFETY: `scores` has `term_count` elements.
+            unsafe { *scores.get_unchecked_mut(term_idx) = Some(score) };
+
+            vacant_entry.insert(scores);
+
+            true
+        }
+    }
+}

core/src/executor/suggest.rs

@@ -0,0 +1,42 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+use super::types::QuerySearchLimit;
+use crate::lexer::preprocessor::PreprocessorOutput;
+use crate::store::StoreItemPart;
+
+impl super::Executor {
+    pub fn suggest(
+        &self,
+        collection: StoreItemPart,
+        bucket: StoreItemPart,
+        input: PreprocessorOutput,
+        limit: QuerySearchLimit,
+    ) -> Result<Option<impl ExactSizeIterator<Item = String> + DoubleEndedIterator>, ()> {
+        // Important: acquire graph access read lock, and reference it in context. This \
+        //   prevents the graph from being erased while using it in this block.
+        let _fst_read_guard = self.fst_pool.lock_read_access();
+
+        if let Ok(fst_store) = self.fst_pool.acquire(collection, bucket) {
+            let mut tokens = input.tokens();
+
+            if let (Some(token), None) = (tokens.next(), tokens.next()) {
+                let len = token.as_original().len();
+                let term = token.into_normalized();
+
+                tracing::debug!("running suggest on word: {term:?}");
+
+                return match fst_store.suggest_words(term, len, limit as usize, None) {
+                    Some(words) => Ok(Some(words.map(|(k, _)| k))),
+                    None => Ok(None),
+                };
+            }
+        }
+
+        Err(())
+    }
+}

core/src/store/fst/mod.rs

@@ -0,0 +1,502 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+mod backup;
+mod pool;
+mod util;
+
+use std::fmt;
+use std::sync::{Arc, RwLock};
+use std::time::SystemTime;
+
+use fst::{IntoStreamer as _, Streamer as _};
+use hashbrown::HashSet;
+use regex_syntax::escape as regex_escape;
+
+use crate::lexer::ranges::LexerRegexRange;
+
+use super::generic::*;
+
+pub use self::pool::{FstStoreId, FstStorePool};
+use self::util::*;
+
+pub struct FstStore {
+    graph: fst::Set,
+    target: FstStoreId,
+    pending: FstStorePending,
+    last_used: Arc<RwLock<SystemTime>>,
+    last_consolidated: Arc<RwLock<SystemTime>>,
+    graph_consolidate: Arc<RwLock<HashSet<FstStoreId>>>,
+    // NOTE: This shouldn’t be here, but until a big rewrite let’s not care.
+    action_config: FstStoreActionConfig,
+}
+
+#[derive(Default)]
+pub struct FstStorePending {
+    pop: Arc<RwLock<HashSet<Vec<u8>>>>,
+    push: Arc<RwLock<HashSet<Vec<u8>>>>,
+}
+
+pub struct FstStoreActionBuilder<'build> {
+    pub fst_store_config: &'build crate::config::FstStoreConfig,
+}
+
+type FstStoreAtom = u32;
+
+pub struct FstStoreMisc;
+
+#[derive(Copy, Clone)]
+enum FstStorePathMode {
+    Permanent,
+    Temporary,
+    Backup,
+}
+
+impl FstStorePathMode {
+    fn extension(&self) -> &'static str {
+        match self {
+            FstStorePathMode::Permanent => ".fst",
+            FstStorePathMode::Temporary => ".fst.tmp",
+            FstStorePathMode::Backup => ".fst.bck",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct FstStoreActionConfig {
+    pub prefix_matching_enabled: bool,
+    pub fuzzy_matching_enabled: bool,
+}
+
+impl Default for FstStoreActionConfig {
+    fn default() -> Self {
+        Self {
+            prefix_matching_enabled: true,
+            fuzzy_matching_enabled: true,
+        }
+    }
+}
+
+const WORD_LIMIT_LENGTH: usize = 40;
+
+impl FstStore {
+    pub fn cardinality(&self) -> usize {
+        self.graph.len()
+    }
+
+    pub fn as_stream(&self) -> fst::set::Stream<'_> {
+        self.graph.into_stream()
+    }
+
+    pub fn lookup_begins_(&self, word: &str) -> Result<fst::set::Stream<'_, fst_regex::Regex>, ()> {
+        // NOTE: This regex maps over an unicode range, for speed reasons at scale.
+        //   We found out that the 'match any' syntax ('.*') was super-slow. Using the restrictive
+        //   syntax below divided the cost of e.g. a search query by 2. The regex below has been
+        //   found out to be nearly zero-cost to compile and execute, for whatever reason.
+        // Regex format: '{escaped_word}([{unicode_range}]*)'
+        let mut regex_str = regex_escape(word);
+
+        regex_str.push('(');
+
+        LexerRegexRange::from(word)
+            .unwrap_or_default()
+            .write_to(&mut regex_str)
+            // Regex write failed? (this should not happen)
+            .map_err(|error| tracing::error!(
+                "Could not lookup word in fst via 'begins': {word:?} because regex write failed: {error:?}"
+            ))?;
+
+        regex_str.push_str("*)");
+
+        // Proceed word lookup.
+        tracing::debug!("Looking-up word in fst via 'begins': {word:?} with regex: {regex_str:?}");
+
+        let regex = fst_regex::Regex::new(&regex_str).map_err(|_error| ())?;
+
+        Ok(self.graph.search(regex).into_stream())
+    }
+
+    pub fn lookup_typos_(
+        &self,
+        word: &str,
+        typo_factor: u32,
+    ) -> Result<fst::set::Stream<'_, fst_levenshtein::Levenshtein>, ()> {
+        tracing::debug!(
+            "Looking-up word in fst via 'typos': {word:?} with typo factor: {typo_factor:?}"
+        );
+
+        let fuzzy = fst_levenshtein::Levenshtein::new(word, typo_factor).map_err(|_error| ())?;
+
+        Ok(self.graph.search(fuzzy).into_stream())
+    }
+
+    pub fn should_consolidate(&self) {
+        let id = self.target;
+
+        // Check if not already scheduled.
+        if self.graph_consolidate.read().unwrap().contains(&id) {
+            tracing::debug!("Graph consolidation already scheduled on pool: {id}");
+            return;
+        };
+
+        // Schedule target for next consolidation tick (i.e. collection + bucket tuple).
+        self.graph_consolidate.write().unwrap().insert(id);
+
+        // Bump “last consolidated” time, effectively de-bouncing consolidation
+        // to a fixed and predictable tick time in the future.
+        let mut last_consolidated_value = self.last_consolidated.write().unwrap();
+
+        *last_consolidated_value = SystemTime::now();
+
+        // Perform an early drop of the lock (frees up write lock early).
+        drop(last_consolidated_value);
+
+        tracing::info!("Graph consolidation scheduled on pool: {id}");
+    }
+}
+
+impl StoreGeneric for FstStore {
+    fn ref_last_used(&self) -> &RwLock<SystemTime> {
+        &self.last_used
+    }
+}
+
+impl FstStore {
+    pub fn push_word(&self, word: &str, fst_store_config: &crate::config::FstStoreConfig) -> bool {
+        // Word over limit? (abort, the FST does not perform well over large words)
+        if Self::word_over_limit(word) {
+            return false;
+        }
+
+        let word_bytes = word.as_bytes();
+
+        // Nuke word from 'pop' set? (void a previous un-consolidated commit)
+        if self.pending.pop.read().unwrap().contains(word_bytes) {
+            self.pending.pop.write().unwrap().remove(word_bytes);
+        }
+
+        // Add word in 'push' set? (only if word is not in FST)
+        // NOTE: also check whether FST is over limits or not from there, to avoid
+        //   stacking words that could never be consolidated to final FST anyway.
+        let graph_fst = self.graph.as_fst();
+
+        if self.graph.contains(&word) {
+            return false;
+        }
+
+        if check_over_limits(graph_fst.size(), graph_fst.len(), &fst_store_config.graph) {
+            return false;
+        }
+
+        {
+            let pending_push_guard = self.pending.push.read().unwrap();
+
+            if pending_push_guard.contains(word_bytes)
+                || pending_push_guard.len() >= fst_store_config.graph.max_words
+            {
+                return false;
+            }
+        }
+
+        (self.pending.push.write().unwrap()).insert(word_bytes.to_vec());
+
+        self.should_consolidate();
+
+        true
+    }
+
+    pub fn pop_word(&self, word: &str) -> bool {
+        // Word over limit? (abort, the FST does not perform well over large words)
+        if Self::word_over_limit(word) {
+            return false;
+        }
+
+        let word_bytes = word.as_bytes();
+
+        // Nuke word from 'push' set? (void a previous un-consolidated commit)
+        if self.pending.push.read().unwrap().contains(word_bytes) {
+            self.pending.push.write().unwrap().remove(word_bytes);
+        }
+
+        if !self.graph.contains(word_bytes) {
+            return false;
+        }
+
+        // Add word in 'pop' set? (only if word is in FST)
+        if self.pending.pop.read().unwrap().contains(word_bytes) {
+            return false;
+        }
+
+        (self.pending.pop.write().unwrap()).insert(word_bytes.to_vec());
+
+        self.should_consolidate();
+
+        true
+    }
+
+    pub fn suggest_words(
+        &self,
+        from_word: &str,
+        // Length before stemming. Useful to apply fuzzy matching rules based
+        // on user input.
+        original_word_len: usize,
+        limit: usize,
+        max_typo_factor: Option<u32>,
+    ) -> Option<impl ExactSizeIterator<Item = (String, u16)> + DoubleEndedIterator + use<>> {
+        use indexmap::IndexMap;
+
+        // Word over limit? (abort, the FST does not perform well over large words)
+        if Self::word_over_limit(from_word) {
+            return None;
+        }
+
+        let mut found_words: IndexMap<String, u16> = IndexMap::with_capacity(limit);
+
+        if self.action_config.prefix_matching_enabled {
+            // Try to complete provided word
+            if let Some(stream) = self.lookup_begins(from_word, original_word_len) {
+                for (word, score) in stream {
+                    if found_words.contains_key(&word) {
+                        continue;
+                    }
+
+                    found_words.insert(word, score);
+
+                    // Requested limit reached? Stop there.
+                    if found_words.len() >= limit {
+                        break;
+                    }
+                }
+            }
+        }
+
+        // Try to fuzzy-suggest other words? (e.g. correct typos)
+        if self.action_config.fuzzy_matching_enabled && found_words.len() < limit {
+            // Allow more typos in word as the word gets longer, up to a maximum limit
+            let max_typo_factor = max_typo_factor.unwrap_or(typo_factor(original_word_len));
+            let mut typo_factor = 1u32;
+
+            // TODO: Rework the Levenshtein query feature to avoid repeating
+            //   the same query over and over again. Maybe try to see if
+            //   `fst_levenshtein` can return distances in its response.
+            while found_words.len() < limit && typo_factor <= max_typo_factor {
+                let Some(stream) = self.lookup_typos(from_word, typo_factor) else {
+                    break;
+                };
+
+                for (word, score) in stream {
+                    if found_words.contains_key(&word) {
+                        continue;
+                    }
+
+                    found_words.insert(word, score);
+
+                    // Requested limit reached? Stop there.
+                    if found_words.len() >= limit {
+                        break;
+                    }
+                }
+
+                typo_factor += 1;
+            }
+        }
+
+        if !found_words.is_empty() {
+            Some(found_words.into_iter())
+        } else {
+            None
+        }
+    }
+
+    pub fn lookup_begins(
+        &self,
+        word: &str,
+        // Length before stemming. Useful to calculate correct score.
+        original_word_len: usize,
+    ) -> Option<impl Iterator<Item = (String, u16)>> {
+        // Word over limit? (abort, the FST does not perform well over large words)
+        if Self::word_over_limit(word) {
+            return None;
+        }
+
+        if !self.action_config.prefix_matching_enabled {
+            return None;
+        }
+
+        let Ok(stream) = self.lookup_begins_(word) else {
+            return None;
+        };
+
+        tracing::debug!(?word, "looking up for word in 'begins' fst stream");
+
+        Some(FstStreamIterator(stream).map(move |word| {
+            // WARN: Calculating distance to original word length might
+            //   yield weird results when combines with stemming.
+            let distance: usize = original_word_len.abs_diff(word.len());
+            let score = u16::try_from(distance).unwrap_or(u16::MAX);
+            (word, score)
+        }))
+    }
+
+    pub fn lookup_typos(
+        &self,
+        word: &str,
+        typo_factor: u32,
+    ) -> Option<impl Iterator<Item = (String, u16)>> {
+        if !self.action_config.fuzzy_matching_enabled {
+            return None;
+        }
+
+        let Ok(stream) = self.lookup_typos_(word, typo_factor) else {
+            return None;
+        };
+
+        tracing::debug!(
+            ?word,
+            typo_factor,
+            "looking up for word in 'typos' fst stream"
+        );
+
+        // NOTE: Returning the same score for every word works only
+        //   because we re-run `lookup_typos` for increasingly
+        //   larger typo factors and do not re-insert existing
+        //   values. As explained in previous TODO, we should try
+        //   to get the real distance back from `fst_levenshtein`.
+        let score = u16::try_from(typo_factor).unwrap_or(u16::MAX);
+
+        Some(FstStreamIterator(stream).map(move |word| (word, score)))
+    }
+
+    pub fn list_words(&self, limit: usize, offset: usize) -> Result<Vec<String>, ()> {
+        let stream = self.as_stream();
+
+        // Enumerate words from FST stream.
+        match stream
+            .into_strs()
+            .map(|words| words.into_iter().skip(offset).take(limit).collect())
+        {
+            Err(err) => {
+                tracing::debug!("conversion of stream failed: {err:?}");
+                Err(())
+            }
+            Ok(words) => Ok(words),
+        }
+    }
+
+    pub fn count_words(&self) -> usize {
+        self.cardinality()
+    }
+
+    fn word_over_limit(word: &str) -> bool {
+        if word.len() > WORD_LIMIT_LENGTH {
+            tracing::debug!("got over-limit fst word: {word:?}");
+
+            true
+        } else {
+            false
+        }
+    }
+}
+
+/// Allow more typos in word as the word gets longer, up to a maximum limit.
+pub(crate) fn typo_factor(word_len: usize) -> u32 {
+    match word_len {
+        1..=3 => 0,
+        4..=6 => 1,
+        7..=9 => 2,
+        _ => 3,
+    }
+}
+
+// MARK: - Helpers
+
+#[repr(transparent)]
+struct FstStreamIterator<'a, A: fst::Automaton>(fst::set::Stream<'a, A>);
+
+impl<'a, A: fst::Automaton> Iterator for FstStreamIterator<'a, A> {
+    type Item = String;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.0.next() {
+            Some(bytes) => match str::from_utf8(bytes) {
+                Ok(str) => Some(str.to_owned()),
+                Err(_) => None,
+            },
+            None => None,
+        }
+    }
+}
+
+// MARK: - Tests
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    pub(in crate::store::fst) fn test_fst_pool() -> FstStorePool {
+        let fst_store_config = test_fst_store_config();
+
+        FstStorePool::new(fst_store_config, Default::default())
+    }
+
+    pub(in crate::store::fst) fn test_fst_store_config() -> Arc<crate::config::FstStoreConfig> {
+        Arc::new(
+            config::Config::builder()
+                .add_source(config::File::from_str(
+                    crate::config::tests::defaults_toml(),
+                    config::FileFormat::Toml,
+                ))
+                .build()
+                .unwrap()
+                .get::<crate::config::FstStoreConfig>("store.fst")
+                .unwrap(),
+        )
+    }
+}
+
+// MARK: - Boilerplate
+
+impl fmt::Debug for FstStore {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use crate::util::fmt::AsPrettyRwLock;
+
+        // NOTE: Deconstructing to future-proof this function.
+        let Self {
+            graph,
+            target,
+            pending,
+            last_used,
+            last_consolidated,
+            graph_consolidate,
+            action_config,
+        } = self;
+
+        f.debug_struct("FstStore")
+            .field("graph", graph)
+            .field("target", target)
+            .field("pending", pending)
+            .field("last_used", &AsPrettyRwLock(last_used))
+            .field("last_consolidated", &AsPrettyRwLock(last_consolidated))
+            .field("graph_consolidate", &AsPrettyRwLock(graph_consolidate))
+            .field("action_config", action_config)
+            .finish()
+    }
+}
+
+impl fmt::Debug for FstStorePending {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use crate::util::fmt::AsPrettyRwLock;
+
+        // NOTE: Deconstructing to future-proof this function.
+        let Self { pop, push } = self;
+
+        f.debug_struct("FstStorePending")
+            .field("pop", &AsPrettyRwLock(pop))
+            .field("push", &AsPrettyRwLock(push))
+            .finish()
+    }
+}

core/tests/common/config.rs

@@ -0,0 +1,80 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+use crate::common::util::unique_hex;
+
+pub fn make_config(toml: &str) -> sonic::Config {
+    let raw_config: config::Config = config::Config::builder()
+        .add_source(config::File::from_str(toml, config::FileFormat::Toml))
+        .build()
+        .expect("error reading config");
+
+    // Parse configuration.
+    let config = raw_config
+        .try_deserialize::<sonic::Config>()
+        .expect("syntax error in config");
+
+    // Validate configuration.
+    config.validate();
+
+    config
+}
+
+pub fn defaults_toml() -> String {
+    let test_id = unique_hex().unwrap();
+
+    let test_data_path = std::path::Path::new(env!("CARGO_TARGET_TMPDIR")).join(&test_id);
+    std::fs::create_dir_all(&test_data_path).unwrap();
+
+    let kv_store_path = test_data_path.join("kv-store");
+    std::fs::create_dir(&kv_store_path).unwrap();
+    let fst_store_path = test_data_path.join("fst-store");
+    std::fs::create_dir(&fst_store_path).unwrap();
+
+    format!(
+        r#"
+        [normalization]
+        unicode_normalization = "none"
+        diacritic_folding_enabled = true
+        stemming_enabled = false
+
+        [tokenization]
+        detect_special_patterns = true
+        compat_split_special_patterns = false
+
+        [stopwords]
+        allow = []
+        deny = []
+
+        [search]
+        query_limit_default = 10
+        query_limit_maximum = 100
+        query_alternates_try = 4
+        query_minimum_term_idf_default = 0.1
+        query_minimum_term_idf_minimum_object_count = 100
+        suggest_limit_default = 5
+        suggest_limit_maximum = 20
+        list_limit_default = 100
+        list_limit_maximum = 500
+
+        [store.kv]
+        path = {kv_store_path:?}
+        retain_word_objects = 1000
+        pool.inactive_after = 1800
+        database.flush_after = 900
+        database.compression_type = "zstd"
+        database.parallelism = 2
+        database.write_ahead_log = true
+
+        [store.fst]
+        path = {fst_store_path:?}
+        pool.inactive_after = 300
+        graph.consolidate_after = 180
+        graph.max_size = 2048
+        graph.max_words = 250000
+        "#,
+    )
+}

core/tests/common/mod.rs

@@ -0,0 +1,171 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+#![allow(unused)]
+
+pub mod config;
+mod executor;
+mod logging;
+mod meta_tests;
+pub mod util;
+
+pub use self::executor::*;
+pub(crate) use self::item_ref::*;
+pub use self::logging::*;
+pub(crate) use self::meta_tests::*;
+pub(crate) use self::util::assert_contains;
+
+// NOTE: Using macros instead of functions so `unwrap`s point to the call site
+//   (helps debugging).
+pub(crate) mod item_ref {
+    macro_rules! collection_ref {
+        ($collection:expr) => {
+            sonic::store::StoreItemBuilder::from_depth_1($collection).unwrap()
+        };
+    }
+    pub(crate) use collection_ref;
+
+    macro_rules! bucket_ref {
+        ($collection:expr, $bucket:expr) => {
+            sonic::store::StoreItemBuilder::from_depth_2($collection, $bucket).unwrap()
+        };
+    }
+    pub(crate) use bucket_ref;
+
+    macro_rules! object_ref {
+        ($collection:expr, $bucket:expr, $object:expr) => {
+            sonic::store::StoreItemBuilder::from_depth_3($collection, $bucket, $object).unwrap()
+        };
+    }
+    pub(crate) use object_ref;
+}
+
+macro_rules! exec {
+    ($executor:ident -> PUSH $collection:tt $bucket:tt $oid:tt $text:tt $(LANG($lang:expr))? NEW) => {
+        exec!(internal_ $executor -> PUSH $collection $bucket $oid $text true $(LANG($lang))?)
+    };
+    ($executor:ident -> PUSH $collection:tt $bucket:tt $oid:tt $text:tt $(LANG($lang:expr))?) => {
+        exec!(internal_ $executor -> PUSH $collection $bucket $oid $text false $(LANG($lang))?)
+    };
+    (internal_ $executor:ident -> PUSH $collection:tt $bucket:tt $oid:tt $text:tt $assume_new:ident $(LANG($lang:expr))?) => {{
+        #[rustfmt::skip]
+        $executor.log(format!(
+            "PUSH {:?} {:?} {:?} {:?}{}{}",
+            $collection, $bucket, $oid, $text, exec!(internal_ lang_txt $($lang)?), if $assume_new { " NEW" } else { "" }
+        ));
+        let (c, b, o) = crate::common::object_ref!($collection, $bucket, $oid);
+        let preprocessor = sonic::lexer::preprocessor::Preprocessor::new(
+            $executor.app_conf.tokenization,
+            $executor.app_conf.normalization,
+            $executor.app_conf.stopwords.clone(),
+            true,
+            true,
+        );
+        $executor
+            .push(
+                c, b, o,
+                preprocessor.preprocess($text, exec!(internal_ lang $($lang)?)),
+                $assume_new,
+            )
+            .unwrap()
+    }};
+
+    ($executor:ident -> TRIGGER consolidate) => {{
+        $executor.log(format!("TRIGGER consolidate"));
+        $executor.fst_pool.consolidate(true, |_| true)
+    }};
+
+    ($executor:ident -> COUNT $collection:tt) => {{
+        $executor.log(format!("COUNT {:?}", $collection));
+        let c = collection_ref!($collection);
+        $executor.countc(c)
+    }};
+
+    ($executor:ident -> COUNT $collection:tt $bucket:tt) => {{
+        $executor.log(format!("COUNT {:?} {:?}", $collection, $bucket));
+        let (c, b) = bucket_ref!($collection, $bucket);
+        $executor.legacy_countb(c, b)
+    }};
+
+    ($executor:ident -> COUNTB $collection:tt $bucket:tt) => {{
+        $executor.log(format!("COUNTB {:?} {:?}", $collection, $bucket));
+        let (c, b) = bucket_ref!($collection, $bucket);
+        $executor.countb(c, b)
+    }};
+
+    ($executor:ident -> COUNT $collection:tt $bucket:tt $oid:tt) => {{
+        $executor.log(format!("COUNT {:?} {:?} {:?}", $collection, $bucket, $oid));
+        let (c, b, o) = object_ref!($collection, $bucket, $oid);
+        $executor.counto(c, b, o)
+    }};
+
+    ($executor:ident -> QUERY $collection:tt $bucket:tt $term:tt $(LANG($lang:expr))? $(LIMIT($limit:expr))?) => {{
+        #[rustfmt::skip]
+        $executor.log(format!(
+            "QUERY {:?} {:?} {:?}{}{}",
+            $collection, $bucket, $term, exec!(internal_ lang_txt $($lang)?), exec!(internal_ limit_txt $($limit)?)
+        ));
+        let (c, b) = crate::common::bucket_ref!($collection, $bucket);
+        let preprocessor = sonic::lexer::preprocessor::Preprocessor::new(
+            $executor.app_conf.tokenization,
+            $executor.app_conf.normalization,
+            $executor.app_conf.stopwords.clone(),
+            true,
+            true,
+        );
+        $executor
+            .search(
+                c, b,
+                preprocessor.preprocess($term, exec!(internal_ lang $($lang)?)),
+                exec!(internal_ limit $($limit)?),
+                0,
+            )
+            .expect("QUERY should succeed")
+    }};
+
+    ($executor:ident -> LIST $collection:tt $bucket:tt $(LIMIT($limit:expr))?) => {{
+        $executor.log(format!("LIST {:?} {:?}", $collection, $bucket));
+        let (c, b) = crate::common::bucket_ref!($collection, $bucket);
+        $executor
+            .list(
+                c, b,
+                exec!(internal_ limit $($limit)?),
+                0,
+            )
+            .expect("LIST should succeed")
+    }};
+
+    ($executor:ident -> FLUSHC $collection:tt) => {{
+        $executor.log(format!("FLUSHC {:?}", $collection));
+        let c = crate::common::collection_ref!($collection);
+        $executor.flushc(c).unwrap()
+    }};
+
+    ($executor:ident -> FLUSHB $collection:tt $bucket:tt) => {{
+        $executor.log(format!("FLUSHB {:?} {:?}", $collection, $bucket));
+        let (c, b) = crate::common::bucket_ref!($collection, $bucket);
+        $executor.flushb(c, b).unwrap()
+    }};
+
+    ($executor:ident -> FLUSHO $collection:tt $bucket:tt $oid:tt) => {{
+        $executor.log(format!("FLUSHO {:?} {:?} {:?}", $collection, $bucket, $oid));
+        let (c, b, o) = crate::common::object_ref!($collection, $bucket, $oid);
+        $executor.flusho(c, b, o).unwrap()
+    }};
+
+    (internal_ lang) => { None };
+    (internal_ lang $lang:expr) => { whatlang::Lang::from_code($lang) };
+
+    (internal_ lang_txt) => { "" };
+    (internal_ lang_txt $lang:expr) => { format!(" LANG({})", $lang) };
+
+    (internal_ limit) => { sonic::executor::QuerySearchLimit::MAX };
+    (internal_ limit $limit:expr) => { $limit };
+
+    (internal_ limit_txt) => { "" };
+    (internal_ limit_txt $limit:expr) => { format!(" LLIMIT({})", $limit) };
+}
+pub(crate) use exec;

core/tests/typo_correction.rs

@@ -0,0 +1,85 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+//! Feature: Typo correction
+//!
+//! A “typo factor” is chosen based on the token’s length. Those tests ensure
+//! typo correction works, and ensure no regression in the quality of the
+//! results.
+//! At the moment of this writing, the mapping function is the following:
+//!
+//! ```
+//! let mut typo_factor = match word.len() {
+//!     1..=3 => 0,
+//!     4..=6 => 1,
+//!     7..=9 => 2,
+//!     _ => 3,
+//! };
+//! ```
+
+mod common;
+
+use crate::common::*;
+
+/// This test sentence contains words that are 3–10 characters-long, while
+/// not being stopwords. It’s not realistic but not having stopwords avoids
+/// false positives in tests.
+const ASTRONOMY_WORDS: &str = "sun moon comet nebula pulsars asteroid satellite spacecraft";
+
+/// Search should allow a certain number of typos (depending on token length).
+///
+/// This covers missing letters, inverted letters and additional letters.
+/// The underlying algorithm used is the Levenshtein distance, so this test is
+/// based on that knowledge.
+#[test]
+fn test_search_allows_typos() {
+    // NOTE: Need to make language explicit because of
+    //   <https://github.com/valeriansaliou/sonic/issues/322#issuecomment-4638688602>.
+    //   Will be fixed separately.
+    #[rustfmt::skip]
+    test_ingest_then_query!(push: ASTRONOMY_WORDS [ensure_no_stopword] LANG("eng"), query: [
+        // 3-letter word, distance = 1.
+        ("sum", false),
+        // 4-letter word, distance = 1.
+        ("ssun", true), // sun
+        ("noon", true), // moon
+        // 4-letter word, distance = 2.
+        ("commit", false),
+        // 6-letter word, distance = 1.
+        ("nzbula", true), // nebula
+        // 6-letter word, distance = 2.
+        ("nzbala", false),
+        // 7-letter word, distance = 2.
+        ("plusars", true), // pulsars
+        // 9-letter word, distance = 2.
+        ("saetllite", true), // satellite
+        // 9-letter word, distance = 3.
+        ("satemmote", false),
+        // 10-letter word, distance = 3.
+        ("sapcecrzft", true), // spacecraft
+        // 10-letter word, distance = 4.
+        ("sapcecarft", false),
+    ] LANG("eng"));
+}
+
+/// Ensures the order of words in search queries is insignificant.
+#[test]
+#[ignore = "Not supported yet (FIXME)"]
+fn test_search_term_order_insignificant() {
+    #[rustfmt::skip]
+    test_ingest_then_query!(push: ASTRONOMY_WORDS [ensure_no_stopword], query: [
+        ("satellite pulsars nebula", true),
+        (&format!("missing {ASTRONOMY_WORDS}"), true),
+    ]);
+}
+
+#[test]
+#[ignore = "Not supported yet"]
+fn test_chinese_typo_correction() {
+    // We should make sure languages with shorter graphemes get typo correction
+    // too.
+    unimplemented!()
+}

docs/search-results-ranking.md

@@ -0,0 +1,143 @@
+# Sonic search results ranking
+
+In search engines like Sonic, it’s common for results to be ranked using
+[tf-idf] or its “evolution” [BM25]. However, to keep a very small index Sonic
+doesn’t store some information that’s necessary to compute [term frequency], a
+core element of both [tf-idf] and [BM25].
+
+Out of simplicity, before `v1.7.0` Sonic used to return results that contained
+all search terms (with some fuzzy matching), sorted by reverse ingestion order
+(i.e. most recent documents first). While this was good enough for some use
+cases, discrepancies in ingestion and query languages would silently affect
+which words were considered stopwords and prevent some documents[^doc] from
+showing up in results (because of the implicit `AND`)[^1].
+
+[^doc]: Since this document is about information retrieval as a general concept, this document will use the term “document” to refer to Sonic objects as it’s the strandard term.
+[^1]: For example, see [issue #262](https://github.com/valeriansaliou/sonic/issues/262).
+
+In `v1.7.0`, we reworked Sonic’s ranking algorithm to yield better results.
+
+## Ranking algorithm
+
+For each search term, Sonic looks for documents containing exactly that search
+term, then it performs fuzzy matching until enough results are found. For each
+term, Sonic keeps track of the [Levenshtein distance] of the closest word found
+in each document. Once that done, a documents’s score is calculated by summing
+up its scores for each term, and documents are finally sorted by score. When
+ties occur, results are sorted by reverse ingestion order[^rio].
+
+[^rio]: This happens at no cost and is invisible in the code, but happens because it’s the order in which results are yielded by the index (on purpose).
+
+Here is some more detailed and accurate pseudo-code:
+
+```txt
+MISSING_MATCH_SCORE = 100
+
+fn query(query, app_config) -> [doc] {
+  limit = app_config.store.kv.retain_word_objects
+  mut alternates_try = app_config.search.query_alternates_try
+
+  // Updates the document’s score for a given term.
+  // Returns whether the document was inserted or not.
+  fn update_score(mut results_matrix, doc, score, term_idx, term_count) -> bool {
+    if results_matrix contains doc:
+      // Update score to keep minimum between existing and new score.
+      results_matrix[doc][term_idx] = min(results_matrix[doc][term_idx], score)
+      return false
+    else:
+      // By default, assign a high score to each term.
+      mut scores = [MISSING_MATCH_SCORE; term_count]
+      scores[term_idx] = score
+      results_matrix[doc] = scores
+      return true
+  }
+
+  mut results_matrix = []
+
+  // Look for exact matches.
+  for (index, term) in normalized(query):
+    for doc in find_exact_matches(term):
+      if len(results_matrix) < limit:
+        score = 0
+        update_score(results_matrix, doc, score, index, len(normalized(query)))
+
+  // Look for words containing `term` as prefix.
+  if len(results_matrix) < limit && alternates_try > 0:
+    for (index, term) in normalized(query):
+      for suggested_term in lookup_begins(term):
+        for doc in limit(find_exact_matches(term), alternates_try):
+          if len(results_matrix) < limit:
+            // Compute Levenshtein distance.
+            score = len(suggested_term) - len(term)
+            if update_score(results_matrix, doc, score, index, len(normalized(query))):
+              alternates_try -= 1
+
+  fn typo_factor(word_len) -> u32 {
+    if      word_len <= 3: return 0
+    else if word_len <= 6: return 1
+    else if word_len <= 9: return 2
+    else                 : return 3
+  }
+
+  // Look for words like `term` (fuzzy matching).
+  if len(results_matrix) < limit && alternates_try > 0:
+    for (index, term) in normalized(query):
+      max_typo_factor = typo_factor(len(term))
+      mut typo_factor = 1
+
+      // Iterate on increasingly larger typo factors (Levenshtein distances).
+      while len(results_matrix) < limit && typo_factor <= max_typo_factor:
+        for (suggested_term, lev_distance) in lookup_typos(term, typo_factor):
+          for doc in limit(find_exact_matches(term), alternates_try):
+            if len(results_matrix) < limit:
+              // Save Levenshtein distance as score.
+              if update_score(results_matrix, doc, lev_distance, index, len(normalized(query))):
+                alternates_try -= 1
+        typo_factor += 1
+
+  // Computes final score for each document, summing individual term scores.
+  // NOTE: Takes missing matches into account thanks to `MISSING_MATCH_SCORE`.
+  fn flatten_matrix(results_matrix) -> [(doc, score)] {
+    mut list = []
+
+    for (doc, scores) in results_matrix:
+      list += (doc, sum(scores))
+
+    return list
+  }
+
+  // Creates a B-Tree of `[doc]` keyed by score, then flattens it.
+  fn sort_by_score(results_map) -> [doc] {
+    btree = btree_by_score(results_map)
+    return flatten(btree)
+  }
+
+  return sort_by_score(flatten_matrix(results_matrix))
+}
+```
+
+### Algorithm properties
+
+- At most `store.kv.retain_word_objects` are returned.
+- Fuzzy matching yields at most `search.query_alternates_try` results.
+- Memory usage is linear, proportional to the number of results.
+- Although the pseudo-code doesn’t show it, most operations are lazy, meaning
+  only useful computations are executed.
+- Scoring is calculated using integer values only, without a single floating
+  point computation, division, square root or any other expensive operation.
+
+### Possible future improvements
+
+- Assign a weight to each term, based on [inverse document frequency], to give
+  more precedence to rare terms.
+  - This would imply making some floating point operations, but they’d only be
+    `O(len(results))`, with idf computation being `O(1)`.
+  - This could be done in parallel from lookups, as it is independent.
+- Parallelize some operations (e.g. each term scoring)?
+  - Overhead might be higher than gain, we’d have to benchmark.
+
+[tf-idf]: https://en.wikipedia.org/wiki/Tf%E2%80%93idf
+[BM25]: https://en.wikipedia.org/wiki/Okapi_BM25 "“Okapi BM25” on Wikipedia"
+[term frequency]: https://en.wikipedia.org/wiki/Tf%E2%80%93idf#Term_frequency "“Term frequency” in “tf-idf” on Wikipedia"
+[inverse document frequency]: https://en.wikipedia.org/wiki/Tf%E2%80%93idf#Inverse_document_frequency "“Inverse document frequency” in “tf-idf” on Wikipedia"
+[Levenshtein distance]: https://en.wikipedia.org/wiki/Levenshtein_distance "“Levenshtein distance” on Wikipedia"

server/src/config.rs

@@ -0,0 +1,325 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+use std::sync::Arc;
+
+use crate::DEFAULT_CONFIG_FILE_PATHS;
+
+pub fn defaults_toml() -> &'static str {
+    r#"
+    [server]
+    log_level = "warn"
+
+    [channel]
+    inet = "[::1]:1491"
+    tcp_timeout = 300
+
+    [normalization]
+    # TODO(major): Enable `"nfkc"` by default.
+    unicode_normalization = "none"
+    # TODO(major): Enable by default.
+    diacritic_folding_enabled = false
+    stemming_enabled = false
+
+    [tokenization]
+    detect_special_patterns = true
+    # TODO(major): Disable by default.
+    compat_split_special_patterns = true
+
+    [stopwords]
+    allow = []
+    deny = []
+
+    [search]
+    query_limit_default = 10
+    query_limit_maximum = 100
+    query_alternates_try = 4
+    query_minimum_term_idf_default = 0.1
+    query_minimum_term_idf_minimum_object_count = 100
+    suggest_limit_default = 5
+    suggest_limit_maximum = 20
+    list_limit_default = 100
+    list_limit_maximum = 500
+
+    [store.kv]
+    path = "./data/store/kv/"
+    retain_word_objects = 1000
+    pool.inactive_after = 1800
+    database.flush_after = 900
+    database.compression_type = "zstd"
+    database.parallelism = 2
+    # database.write_buffer_size = 16384 # Default handled via serde
+    database.write_ahead_log = true
+
+    [store.fst]
+    path = "./data/store/fst/"
+    pool.inactive_after = 300
+    graph.consolidate_after = 180
+    graph.max_size = 2048
+    graph.max_words = 250000
+    "#
+}
+
+impl Config {
+    pub fn parse(source: impl ::config::Source + Send + Sync + 'static) -> Self {
+        // Read configuration.
+        let raw_config: config::Config = config::Config::builder()
+            // Start from defaults.
+            .add_source(config::File::from_str(
+                defaults_toml(),
+                config::FileFormat::Toml,
+            ))
+            // Merge static configuration (from file).
+            .add_source(source)
+            // Merge environment overrides.
+            .add_source(
+                config::Environment::with_prefix("SONIC")
+                    .separator("__")
+                    .prefix_separator("_"),
+            )
+            .build()
+            .expect("error reading config");
+
+        // `#[serde(flatten)]` breaks type coercion from the `config` crate,
+        // so we can’t have `crate::Config` define
+        // `#[serde(flatten)] sonic: sonic::Config`. It’s a bit dirty but at
+        // least we don’t have to manually add custom deserialization logic to
+        // all non-string fields (in the core!).
+        #[derive(serde::Deserialize)]
+        pub struct ServerConfigTemp {
+            pub channel: ChannelConfig,
+            pub server: ServerConfig,
+        }
+
+        // Parse configuration.
+        let mut server_config = raw_config
+            .clone()
+            .try_deserialize::<ServerConfigTemp>()
+            .expect("syntax error in config");
+        let mut core_config = raw_config
+            .try_deserialize::<sonic::Config>()
+            .expect("syntax error in config");
+
+        back_compat::migrate_channel_search(&mut server_config.channel, &mut core_config);
+
+        // Validate configuration.
+        core_config.validate();
+
+        Config {
+            channel: server_config.channel,
+            server: server_config.server,
+            sonic: Arc::new(core_config),
+        }
+    }
+}
+
+pub fn read_config(custom_path: Option<&str>) -> Config {
+    if let Some(custom_path) = custom_path {
+        // Abort if the user specified a config path that does not exist.
+        if !std::path::Path::new(custom_path).exists() {
+            panic!("Cannot find config file at {custom_path:?}");
+        }
+
+        let path = std::path::absolute(custom_path).unwrap();
+        tracing::debug!("reading config file: {path:?}");
+
+        Config::parse(config::File::from(path).format(config::FileFormat::Toml))
+    } else {
+        for path in DEFAULT_CONFIG_FILE_PATHS {
+            let path = std::path::absolute(path).unwrap();
+            tracing::debug!("looking for config file at {path:?}");
+
+            if path.exists() {
+                tracing::debug!("reading config file: {path:?}");
+
+                return Config::parse(config::File::from(path).format(config::FileFormat::Toml));
+            }
+        }
+
+        tracing::debug!("no config file found in known locations, using defaults only");
+
+        Config::parse(config::File::from_str("", config::FileFormat::Toml))
+    }
+}
+
+pub struct Config {
+    pub channel: ChannelConfig,
+
+    pub server: ServerConfig,
+
+    pub sonic: Arc<sonic::Config>,
+}
+
+#[allow(deprecated)]
+#[derive(serde::Deserialize)]
+pub struct ChannelConfig {
+    #[serde(deserialize_with = "sonic::util::serde::env_var::socket_addr")]
+    pub inet: std::net::SocketAddr,
+
+    pub tcp_timeout: u64,
+
+    #[serde(default, deserialize_with = "sonic::util::serde::env_var::opt_str")]
+    pub auth_password: Option<String>,
+
+    #[deprecated(since = "1.6.0", note = "Use `search` instead of `channel.search`")]
+    #[serde(default)]
+    pub search: Option<back_compat::ChannelSearchConfig>,
+}
+
+#[derive(serde::Deserialize)]
+pub struct ServerConfig {
+    #[serde(deserialize_with = "sonic::util::serde::env_var::str")]
+    pub log_level: String,
+}
+
+#[allow(deprecated)]
+mod back_compat {
+    #[deprecated(since = "1.6.0", note = "Use `search` instead of `channel.search`")]
+    #[derive(serde::Deserialize)]
+    pub struct ChannelSearchConfig {
+        #[serde(default)]
+        pub query_limit_default: Option<u16>,
+
+        #[serde(default)]
+        pub query_limit_maximum: Option<u16>,
+
+        #[serde(default)]
+        pub query_alternates_try: Option<usize>,
+
+        #[serde(default)]
+        pub suggest_limit_default: Option<u16>,
+
+        #[serde(default)]
+        pub suggest_limit_maximum: Option<u16>,
+
+        #[serde(default)]
+        pub list_limit_default: Option<u16>,
+
+        #[serde(default)]
+        pub list_limit_maximum: Option<u16>,
+    }
+
+    // This is dirty, but AFAIK (@RemiBardon) the `config` crate doesn’t
+    // provide a better API and hopefully we won’t have to do this again.
+    pub fn migrate_channel_search(
+        channel: &mut crate::config::ChannelConfig,
+        sonic: &mut sonic::Config,
+    ) {
+        if let Some(search) = channel.search.take() {
+            tracing::warn!(
+                "You’re still using the deprecated `channel.search` key. \
+                Please use `search` instead. \
+                For this run, we will override `search` with `channel.search`."
+            );
+
+            let ChannelSearchConfig {
+                query_limit_default,
+                query_limit_maximum,
+                query_alternates_try,
+                suggest_limit_default,
+                suggest_limit_maximum,
+                list_limit_default,
+                list_limit_maximum,
+            } = search;
+
+            if let Some(query_limit_default) = query_limit_default {
+                sonic.search.query_limit_default = query_limit_default;
+            }
+            if let Some(query_limit_maximum) = query_limit_maximum {
+                sonic.search.query_limit_maximum = query_limit_maximum;
+            }
+            if let Some(query_alternates_try) = query_alternates_try {
+                sonic.search.query_alternates_try = query_alternates_try;
+            }
+            if let Some(suggest_limit_default) = suggest_limit_default {
+                sonic.search.suggest_limit_default = suggest_limit_default;
+            }
+            if let Some(suggest_limit_maximum) = suggest_limit_maximum {
+                sonic.search.suggest_limit_maximum = suggest_limit_maximum;
+            }
+            if let Some(list_limit_default) = list_limit_default {
+                sonic.search.list_limit_default = list_limit_default;
+            }
+            if let Some(list_limit_maximum) = list_limit_maximum {
+                sonic.search.list_limit_maximum = list_limit_maximum;
+            }
+        }
+    }
+
+    #[cfg(test)]
+    #[test]
+    fn test_channel_search_migration() {
+        let old_config_toml = r#"
+        [channel]
+        search.query_limit_default = 42
+        search.query_alternates_try = 42
+        search.query_minimum_term_idf_default = 42
+        search.query_minimum_term_idf_minimum_object_count = 42
+        search.suggest_limit_default = 42
+        search.suggest_limit_maximum = 42
+        search.list_limit_default = 42
+        search.list_limit_maximum = 42
+        "#;
+
+        let config = crate::Config::parse(config::File::from_str(
+            old_config_toml,
+            config::FileFormat::Toml,
+        ));
+
+        assert!(
+            config.channel.search.is_none(),
+            "`channel.search` not emptied"
+        );
+        assert_eq!(
+            config.sonic.search.list_limit_default, 42,
+            "should be overriden"
+        );
+        assert_ne!(
+            config.sonic.search.query_limit_maximum, 42,
+            "should not be overriden"
+        );
+    }
+
+    #[cfg(test)]
+    #[test]
+    fn test_channel_search_migration_mixed() {
+        let old_config_toml = r#"
+        [channel.search]
+        query_limit_default = 42
+        # query_limit_maximum = 42
+        query_alternates_try = 42
+        query_minimum_term_idf_default = 42
+        query_minimum_term_idf_minimum_object_count = 42
+        suggest_limit_default = 42
+        suggest_limit_maximum = 42
+        list_limit_default = 42
+        list_limit_maximum = 42
+
+        [search]
+        query_limit_default = 8
+        query_limit_maximum = 8
+        "#;
+
+        let config = crate::Config::parse(config::File::from_str(
+            old_config_toml,
+            config::FileFormat::Toml,
+        ));
+
+        assert!(
+            config.channel.search.is_none(),
+            "`channel.search` not emptied"
+        );
+        assert_eq!(
+            config.sonic.search.list_limit_default, 42,
+            "should be overriden"
+        );
+        assert_eq!(
+            config.sonic.search.query_limit_maximum, 8,
+            "should not be overriden"
+        );
+    }
+}