Upstream valeriansaliou/sonic state of the touched files before abd3d56935

core/src/config.rs

@@ -0,0 +1,407 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+//! Sonic library configuration.
+//!
+//! It does not include server nor channel configuration, which are specific
+//! to the `sonic-server` binary.
+
+use std::collections::HashSet;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use serde::Deserialize;
+
+use crate::util::serde::env_var;
+
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    pub normalization: NormalizationConfig,
+
+    pub tokenization: TokenizationConfig,
+
+    pub stopwords: StopwordsConfig,
+
+    pub search: SearchConfig,
+
+    pub store: StoreConfig,
+}
+
+impl Config {
+    pub fn validate(&self) {
+        // Check 'write_buffer' for KV
+        if self.store.kv.database.write_buffer_size == Some(0) {
+            panic!("write_buffer for kv must not be zero");
+        }
+
+        // Check 'flush_after' for KV
+        if self.store.kv.database.flush_after >= self.store.kv.pool.inactive_after {
+            panic!("flush_after for kv must be strictly lower than inactive_after");
+        }
+
+        // Check 'consolidate_after' for FST
+        if self.store.fst.graph.consolidate_after >= self.store.fst.pool.inactive_after {
+            panic!("consolidate_after for fst must be strictly lower than inactive_after");
+        }
+    }
+}
+
+/// Configuration group for normalization options (Unicode normalization,
+/// stemming, lemmatization…).
+#[derive(Debug, Deserialize, Clone, Copy)]
+pub struct NormalizationConfig {
+    #[serde(with = "crate::util::serde::none_string_as_none")]
+    pub unicode_normalization: Option<UnicodeNormalization>,
+
+    pub diacritic_folding_enabled: bool,
+
+    #[cfg(feature = "stemming")]
+    pub stemming_enabled: bool,
+}
+
+#[derive(Deserialize, Debug, Clone, Copy)]
+#[serde(rename_all = "snake_case")]
+pub enum UnicodeNormalization {
+    /// Unicode Normalization Form C.
+    Nfc,
+    /// Unicode Normalization Form KC.
+    Nfkc,
+}
+
+/// Configuration group for tokenization options.
+#[derive(Debug, Deserialize, Clone, Copy)]
+pub struct TokenizationConfig {
+    pub detect_special_patterns: bool,
+
+    #[serde(alias = "split_special_patterns")]
+    pub compat_split_special_patterns: bool,
+}
+
+#[derive(Debug, Deserialize, Clone, Default)]
+pub struct StopwordsConfig {
+    #[serde(deserialize_with = "to_stopwords")]
+    pub allow: HashSet<String>,
+
+    #[serde(deserialize_with = "to_stopwords")]
+    pub deny: HashSet<String>,
+}
+
+fn to_stopwords<'de, D>(deserializer: D) -> Result<HashSet<String>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    use unicode_normalization::UnicodeNormalization as _;
+
+    let vec: Vec<Box<str>> = Deserialize::deserialize(deserializer)?;
+    let stopwords_iter = vec.into_iter().map(|s| s.nfkd().to_string());
+    Ok(HashSet::from_iter(stopwords_iter))
+}
+
+#[derive(Debug, Deserialize)]
+pub struct SearchConfig {
+    pub query_limit_default: u16,
+
+    pub query_limit_maximum: u16,
+
+    pub query_alternates_try: usize,
+
+    pub query_minimum_term_idf_default: f32,
+
+    pub query_minimum_term_idf_minimum_object_count: u64,
+
+    pub suggest_limit_default: u16,
+
+    pub suggest_limit_maximum: u16,
+
+    pub list_limit_default: u16,
+
+    pub list_limit_maximum: u16,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct StoreConfig {
+    pub kv: Arc<KvStoreConfig>,
+
+    pub fst: Arc<FstStoreConfig>,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct KvStoreConfig {
+    #[serde(deserialize_with = "env_var::path_buf")]
+    pub path: PathBuf,
+
+    pub retain_word_objects: usize,
+
+    pub pool: KvStorePoolConfig,
+
+    pub database: KvStoreDatabaseConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct KvStorePoolConfig {
+    pub inactive_after: u64,
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct KvStoreDatabaseConfig {
+    pub flush_after: u64,
+
+    pub write_ahead_log: bool,
+
+    /// Whether or not to compress.
+    ///
+    /// Will get overriden if [`compression_type`](Self::compression_type) is
+    /// also specified.
+    #[serde(default)]
+    pub compress: Option<bool>,
+
+    #[serde(default)]
+    pub parallelism: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "max_files")]
+    pub max_open_files: Option<i32>,
+
+    // TODO(major): Make this MB, as in Kvrocks.
+    /// WARN: In KB!
+    #[serde(default = "default_write_buffer_size")]
+    #[serde(alias = "write_buffer")]
+    pub write_buffer_size: Option<usize>,
+
+    #[serde(default)]
+    pub max_write_buffer_number: Option<i32>,
+
+    #[serde(default)]
+    pub min_write_buffer_number: Option<i32>,
+
+    #[serde(default)]
+    pub min_write_buffer_number_to_merge: Option<i32>,
+
+    #[serde(default)]
+    pub block_cache_size: Option<u32>,
+
+    #[serde(default)]
+    pub cache_index_and_filter_blocks: Option<bool>,
+
+    #[serde(default)]
+    #[serde(alias = "compression")]
+    #[serde(deserialize_with = "to_rocksdb_compression_type_opt")]
+    pub compression_type: Option<rocksdb::DBCompressionType>,
+
+    #[serde(default)]
+    #[serde(alias = "wal_compression")]
+    #[serde(deserialize_with = "to_rocksdb_compression_type_opt")]
+    pub wal_compression_type: Option<rocksdb::DBCompressionType>,
+
+    #[serde(default)]
+    pub wal_ttl_seconds: Option<u64>,
+
+    #[serde(default)]
+    pub wal_size_limit_mb: Option<u64>,
+
+    #[serde(default)]
+    pub wal_bytes_per_sync: Option<u64>,
+
+    #[serde(default)]
+    #[serde(deserialize_with = "to_rocksdb_recovery_mode_opt")]
+    pub wal_recovery_mode: Option<rocksdb::DBRecoveryMode>,
+
+    #[serde(default)]
+    pub compression_level: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "compression_start_level")]
+    pub min_level_to_compress: Option<std::ffi::c_int>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_file_num_compaction_trigger")]
+    pub level_zero_file_num_compaction_trigger: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_slowdown_writes_trigger")]
+    pub level_zero_slowdown_writes_trigger: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "level0_stop_writes_trigger")]
+    pub level_zero_stop_writes_trigger: Option<i32>,
+
+    #[serde(default)]
+    pub max_bytes_for_level_base: Option<u64>,
+
+    #[serde(default)]
+    pub max_bytes_for_level_multiplier: Option<f64>,
+
+    #[serde(default)]
+    pub target_file_size_base: Option<u64>,
+
+    #[serde(default)]
+    pub max_background_jobs: Option<i32>,
+
+    #[serde(default)]
+    #[serde(alias = "max_compactions")]
+    pub max_subcompactions: Option<u32>,
+
+    #[serde(default)]
+    pub max_flushes: Option<i32>,
+
+    #[serde(default)]
+    pub stats_dump_period_sec: Option<u32>,
+}
+
+fn default_write_buffer_size() -> Option<usize> {
+    Some(16384)
+}
+
+fn parse_rocksdb_compression_type<E: serde::de::Error>(
+    str: &str,
+) -> Result<rocksdb::DBCompressionType, E> {
+    // NOTE: Some values are not available because not compiled in rocksdb
+    //   (feature flag is off).
+    match str.to_ascii_lowercase().as_str() {
+        "none" => Ok(rocksdb::DBCompressionType::None),
+        // "snappy" => Ok(rocksdb::DBCompressionType::Snappy),
+        // "zlib" => Ok(rocksdb::DBCompressionType::Zlib),
+        // "bz2" => Ok(rocksdb::DBCompressionType::Bz2),
+        // "lz4" => Ok(rocksdb::DBCompressionType::Lz4),
+        // "lz4hc" => Ok(rocksdb::DBCompressionType::Lz4hc),
+        "zstd" => Ok(rocksdb::DBCompressionType::Zstd),
+        _ => Err(serde::de::Error::unknown_variant(
+            str,
+            &[
+                "none",
+                // "snappy",
+                // "zlib",
+                // "bz2",
+                // "lz4",
+                // "lz4hc",
+                "zstd",
+            ],
+        )),
+    }
+}
+
+fn to_rocksdb_compression_type_opt<'de, D>(
+    deserializer: D,
+) -> Result<Option<rocksdb::DBCompressionType>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    let str: Option<String> = Deserialize::deserialize(deserializer)?;
+    str.map(|s| parse_rocksdb_compression_type(&s)).transpose()
+}
+
+fn parse_rocksdb_recovery_mode<E: serde::de::Error>(
+    str: &str,
+) -> Result<rocksdb::DBRecoveryMode, E> {
+    match str.to_ascii_lowercase().as_str() {
+        "tolerate_corrupted_tail_records" | "TolerateCorruptedTailRecords" => {
+            Ok(rocksdb::DBRecoveryMode::TolerateCorruptedTailRecords)
+        }
+        "absolute_consistency" | "AbsoluteConsistency" => {
+            Ok(rocksdb::DBRecoveryMode::AbsoluteConsistency)
+        }
+        "point_in_time" | "PointInTime" => Ok(rocksdb::DBRecoveryMode::PointInTime),
+        "skip_any_corrupted_record" | "SkipAnyCorruptedRecord" => {
+            Ok(rocksdb::DBRecoveryMode::SkipAnyCorruptedRecord)
+        }
+        _ => Err(serde::de::Error::unknown_variant(
+            str,
+            &[
+                "tolerate_corrupted_tail_records",
+                "absolute_consistency",
+                "point_in_time",
+                "skip_any_corrupted_record",
+            ],
+        )),
+    }
+}
+
+fn to_rocksdb_recovery_mode_opt<'de, D>(
+    deserializer: D,
+) -> Result<Option<rocksdb::DBRecoveryMode>, D::Error>
+where
+    D: serde::de::Deserializer<'de>,
+{
+    let str: Option<String> = Deserialize::deserialize(deserializer)?;
+    str.map(|s| parse_rocksdb_recovery_mode(&s)).transpose()
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStoreConfig {
+    #[serde(deserialize_with = "env_var::path_buf")]
+    pub path: PathBuf,
+
+    pub pool: FstStorePoolConfig,
+
+    pub graph: FstStoreGraphConfig,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStorePoolConfig {
+    pub inactive_after: u64,
+}
+
+#[derive(Debug, Deserialize)]
+pub struct FstStoreGraphConfig {
+    pub consolidate_after: u64,
+
+    pub max_size: usize,
+
+    pub max_words: usize,
+}
+
+#[cfg(test)]
+pub(crate) mod tests {
+    pub(crate) fn defaults_toml() -> &'static str {
+        r#"
+        [channel]
+        inet = "[::1]:1491"
+        tcp_timeout = 300
+
+        [normalization]
+        unicode_normalization = "none"
+        diacritic_folding_enabled = true
+        stemming_enabled = false
+
+        [tokenization]
+        detect_special_patterns = true
+        compat_split_special_patterns = false
+
+        [stopwords]
+        allow = []
+        deny = []
+
+        [search]
+        query_limit_default = 10
+        query_limit_maximum = 100
+        query_alternates_try = 4
+        query_minimum_term_idf_default = 0.1
+        query_minimum_term_idf_minimum_object_count = 100
+        suggest_limit_default = 5
+        suggest_limit_maximum = 20
+        list_limit_default = 100
+        list_limit_maximum = 500
+
+        [store.kv]
+        path = "./data/store/kv/"
+        retain_word_objects = 1000
+        pool.inactive_after = 1800
+        database.flush_after = 900
+        database.compression_type = "zstd"
+        database.parallelism = 2
+        database.write_ahead_log = true
+
+        [store.fst]
+        path = "./data/store/fst/"
+        pool.inactive_after = 300
+        graph.consolidate_after = 180
+        graph.max_size = 2048
+        graph.max_words = 250000
+        "#
+    }
+}

core/src/executor/search.rs

@@ -0,0 +1,613 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+use indexmap::IndexMap;
+
+use super::types::{QueryMatchScore, QueryResultScore, QuerySearchLimit, QuerySearchOffset};
+use crate::lexer::itertools::UniqueBy;
+use crate::lexer::preprocessor::{PreprocessorOutput, Token};
+use crate::store::StoreItemPart;
+use crate::store::fst::typo_factor;
+use crate::store::kv::KvStoreActionReadOnly;
+use crate::store::{StoreObjectIid, StoreTermHash};
+use crate::util::hash::NoopU32HasherBuilder;
+
+impl super::Executor {
+    pub fn search(
+        &self,
+        collection: StoreItemPart,
+        bucket: StoreItemPart,
+        input: PreprocessorOutput,
+        limit: QuerySearchLimit,
+        offset: QuerySearchOffset,
+    ) -> Result<Vec<String>, ()> {
+        // Important: acquire database access read lock, and reference it in context. This \
+        //   prevents the database from being erased while using it in this block.
+        let _kv_read_guard = self.kv_pool.lock_read_access();
+        let _fst_read_guard = self.fst_pool.lock_read_access();
+
+        let (Ok(kv_store), Ok(fst_store)) = (
+            self.kv_pool.acquire(false, collection, None, |_| {}),
+            self.fst_pool.acquire(collection, bucket),
+        ) else {
+            return Err(());
+        };
+
+        let Some(kv_store) = kv_store else {
+            tracing::debug!(
+                "collection store does not exist, consider {bucket:?} from {collection:?} empty"
+            );
+            return Ok(vec![]);
+        };
+
+        let (higher_limit, mut alternates_try) = (
+            self.app_conf.store.kv.retain_word_objects,
+            self.app_conf.search.query_alternates_try,
+        );
+
+        let (mut minimum_idf, idf_min_doc_count) = (
+            self.app_conf.search.query_minimum_term_idf_default,
+            (self.app_conf.search).query_minimum_term_idf_minimum_object_count,
+        );
+
+        let (prefix_matching_enabled, fuzzy_matching_enabled) = (
+            self.fst_pool.fst_action_config.prefix_matching_enabled,
+            self.fst_pool.fst_action_config.fuzzy_matching_enabled,
+        );
+
+        // Important: acquire bucket store read lock
+        executor_kv_lock_read!(kv_store);
+
+        let kv_action = kv_store.access_read_only(bucket);
+
+        let document_count = kv_action
+            .get_object_count()
+            .map_err(|error| tracing::warn!("{error:?}"))? as u64;
+
+        if document_count < idf_min_doc_count {
+            tracing::debug!(
+                "ignoring minimum_term_idf ({minimum_idf}) as document_count is too low ({document_count}<{idf_min_doc_count})"
+            );
+            minimum_idf = 0.;
+        }
+
+        // Collect all terms so we know the count right ahead.
+        // PERF: This helps allocating the correct amounts of memory.
+        let tokens: Vec<Token> =
+            UniqueBy::new_with_hasher(input.tokens(), Token::hash, NoopU32HasherBuilder).collect();
+        let term_count = tokens.len();
+
+        // Store scores for each found IID. Results will then be sorted by
+        // score before being returned. Scores are basically the sum of
+        // Levenshtein distances for each term in the query. Lower score
+        // means better result.
+        // NOTE: We use `IndexMap` instead of `HashMap` to preserve
+        //   insertion order, which correlates to reverse data ingestion
+        //   order.
+        // NOTE: `capacity = 24` to reduce initial grows.
+        let mut scoring_matrix: IndexMap<StoreObjectIid, Vec<Option<QueryMatchScore>>> =
+            IndexMap::with_capacity(24usize.min(usize::from(limit)));
+
+        // Look for exact matches.
+        'matches: for (idx, token) in tokens.iter().enumerate() {
+            let term_hash = token.hash();
+            let term = token.as_normalized();
+
+            let mut iids = kv_action
+                .get_term_to_iids(term_hash)
+                .unwrap_or(None)
+                .unwrap_or_default();
+
+            // Look for exact matches normalized differently if the Sonic
+            // index isn’t normalized.
+            if !token.is_special() && self.app_conf.normalization.unicode_normalization.is_none() {
+                use unicode_normalization::UnicodeNormalization as _;
+
+                let mut nfc = kv_action
+                    .get_term_to_iids(StoreTermHash::from(term.nfc().to_string().as_str()))
+                    .unwrap_or(None)
+                    .unwrap_or_default();
+                iids.append(&mut nfc);
+
+                let mut nfd = kv_action
+                    .get_term_to_iids(StoreTermHash::from(term.nfd().to_string().as_str()))
+                    .unwrap_or(None)
+                    .unwrap_or_default();
+                iids.append(&mut nfd);
+            };
+
+            tracing::debug!("got exact search executor iids: {iids:?} for term: {term:?}");
+
+            let document_frequency = document_frequency(term_hash, &kv_action);
+
+            // Filter out minimum IDF.
+            // PERF: Filtering `minimum_idf > 0` to save some computation.
+            if minimum_idf > 0. {
+                let idf = (document_count as f32 / document_frequency as f32).ln();
+                if idf < minimum_idf {
+                    tracing::debug!(
+                        "skipping term {term:?} because idf too low ({idf}<{minimum_idf})"
+                    );
+                    continue;
+                }
+            }
+
+            let bm25_score = bm25_lite_idf(document_count, document_frequency);
+
+            for iid in iids.into_iter() {
+                // Assign a base score of `1` as those are exact matches.
+                let inserted =
+                    update_score(&mut scoring_matrix, iid, 1. * bm25_score, idx, term_count);
+
+                if inserted {
+                    // Higher limit now reached?
+                    // Stop acquiring new suggested IIDs now.
+                    if scoring_matrix.len() >= higher_limit {
+                        tracing::trace!(?term, "got enough completed results for term");
+
+                        break 'matches;
+                    }
+                }
+            }
+        }
+
+        #[cfg(debug_assertions)]
+        tracing::debug!(?scoring_matrix);
+
+        // Look for words containing `term` as prefix.
+        if scoring_matrix.len() < higher_limit && alternates_try > 0 && prefix_matching_enabled {
+            tracing::debug!(
+                "not enough iids were found ({}/{higher_limit}), looking for prefixes",
+                scoring_matrix.len(),
+            );
+
+            'terms: for (idx, token) in tokens.iter().enumerate() {
+                let original_len = token.as_original().len();
+                let term = token.as_normalized();
+
+                let Some(suggestions) = fst_store.lookup_begins(term, original_len) else {
+                    tracing::trace!("did not get any completed word for term {term:?}");
+                    continue 'terms;
+                };
+
+                merge_suggestions(
+                    suggestions.map(|(w, distance)| (w, prefix_score(distance, original_len))),
+                    &mut scoring_matrix,
+                    term,
+                    idx,
+                    term_count,
+                    &kv_action,
+                    &mut alternates_try,
+                    higher_limit,
+                    document_count,
+                    minimum_idf,
+                );
+            }
+        }
+
+        #[cfg(debug_assertions)]
+        tracing::debug!(?scoring_matrix);
+
+        // Look for words like `term` (fuzzy matching).
+        if scoring_matrix.len() < higher_limit && alternates_try > 0 && fuzzy_matching_enabled {
+            tracing::debug!(
+                "not enough iids were found ({}/{higher_limit}), looking for fuzzy matches",
+                scoring_matrix.len(),
+            );
+
+            'terms: for (idx, token) in tokens.iter().enumerate() {
+                let original_word_len = token.as_original().len();
+                let term = token.as_normalized();
+
+                // Skip term if it’s special (we’d want exact matches only).
+                if token.is_special() {
+                    tracing::debug!("skipping fuzzy search for {term:?}: term is special");
+                    continue 'terms;
+                }
+
+                let max_typo_factor = typo_factor(original_word_len);
+                let mut typo_factor = 1u32;
+
+                // TODO: Rework the Levenshtein query feature to avoid repeating
+                //   the same query over and over again. Maybe try to see if
+                //   `fst_levenshtein` can return distances in its response.
+                while alternates_try > 0 && typo_factor <= max_typo_factor {
+                    let Some(suggestions) = fst_store.lookup_typos(term, typo_factor) else {
+                        tracing::trace!("did not get any completed word for term {term:?}");
+                        continue 'terms;
+                    };
+
+                    merge_suggestions(
+                        suggestions
+                            .map(|(w, distance)| (w, typo_score(distance, original_word_len))),
+                        &mut scoring_matrix,
+                        term,
+                        idx,
+                        term_count,
+                        &kv_action,
+                        &mut alternates_try,
+                        higher_limit,
+                        document_count,
+                        minimum_idf,
+                    );
+
+                    typo_factor += 1;
+                }
+            }
+        }
+
+        #[cfg(debug_assertions)]
+        tracing::debug!(?scoring_matrix);
+
+        // Switch to implicit `AND` if query contains a special token.
+        // NOTE: When a user queries for a special token (e.g. UUID),
+        //   they expect only exact matches to be returned. If one term is
+        //   considered special, we drop all results missing at least one
+        //   term. It’s not the most efficient (compared to not storing the
+        //   result in the first place) but it’s an edge case and the cost
+        //   is negligible.
+        let one_term_is_special = tokens.iter().any(Token::is_special);
+        if one_term_is_special {
+            let mut to_remove = Vec::<StoreObjectIid>::new();
+
+            for (&iid, scores) in scoring_matrix.iter() {
+                if scores.iter().any(Option::is_none) {
+                    to_remove.push(iid);
+                }
+            }
+
+            for iid in to_remove {
+                scoring_matrix.swap_remove(&iid);
+            }
+        }
+
+        // Flatten scores, taking into account missing matches (thanks to
+        // `None`).
+        let found_iids = scoring_matrix
+            .into_iter()
+            .map(|(iid, scores)| (iid, overall_score(&scores)));
+
+        // Sort found IIDs.
+        let all_iids = {
+            let mut all_iids = found_iids.collect::<Vec<_>>();
+            all_iids.sort_by(|a, b| a.1.total_cmp(&b.1).reverse());
+            all_iids.into_iter().map(|(iid, _score)| iid)
+        };
+
+        // Resolve OIDs from IIDs
+        // Notice: we also proceed paging from there
+        let (limit_usize, offset_usize) = (limit as usize, offset as usize);
+        let mut result_oids = Vec::with_capacity(limit_usize);
+
+        'paging: for (index, found_iid) in all_iids.skip(offset_usize).enumerate() {
+            // Stop there?
+            if index >= limit_usize {
+                break 'paging;
+            }
+
+            // Read IID-to-OID for this found IID
+            if let Ok(Some(oid)) = kv_action.get_iid_to_oid(found_iid) {
+                result_oids.push(oid);
+            } else {
+                tracing::error!("failed getting search executor iid-to-oid");
+            }
+        }
+
+        tracing::info!("got search executor final oids: {:?}", result_oids);
+
+        Ok(result_oids)
+    }
+}
+
+/// Inversely proportional to `lev_distance / word_len`, decreasing slowly
+/// towards `f(20) = 0.5`. Will never reach `0`.
+fn prefix_score(lev_distance: u16, word_len: usize) -> f32 {
+    // NOTE: Will be `> 1` in practice.
+    let lev_ratio = lev_distance as f32 / word_len as f32;
+
+    // NOTE: `20` means that auto-completed words 20 times longer than the
+    //   original word get a score of `0.5`. It’s just a magic number, it has
+    //   no further meaning. It just feels ok.
+    20. / (20. + lev_ratio)
+}
+
+#[cfg(test)]
+#[test]
+fn test_prefix_score() {
+    // Auto-complete 2 times longer.
+    for n in [2, 4, 8] {
+        assert_eq!(prefix_score(n, n as usize), 0.95238096);
+    }
+
+    // Auto-complete 4 times longer.
+    for n in [2, 4, 8] {
+        assert_eq!(prefix_score(3 * n, n as usize), 0.8695652);
+    }
+
+    // Auto-complete 1 character.
+    assert_eq!(prefix_score(1, 3), 0.9836065);
+    assert_eq!(prefix_score(1, 4), 0.9876543);
+    assert_eq!(prefix_score(1, 5), 0.99009895);
+    assert_eq!(prefix_score(1, 6), 0.9917356);
+
+    // Auto-complete 2 characters.
+    assert_eq!(prefix_score(2, 3), 0.96774197);
+    assert_eq!(prefix_score(2, 4), 0.9756098);
+    assert_eq!(prefix_score(2, 5), 0.98039216);
+    assert_eq!(prefix_score(2, 6), 0.9836065);
+
+    // More auto-complete means lower score.
+    for n in [1, 2, 4, 8] {
+        for word_len in [2, 4, 8, 10] {
+            assert!(prefix_score(n + 1, word_len as usize) < prefix_score(n, word_len as usize));
+        }
+    }
+}
+
+/// Levenshtein distance proportional to word length.
+fn typo_score(lev_distance: u16, word_len: usize) -> f32 {
+    debug_assert!(
+        (lev_distance as usize) < word_len,
+        "{lev_distance} >= {word_len}"
+    );
+
+    // SAFETY: `.min(1)` isn’t strictly necessary as `lev_distance` should
+    //   always be `< word_len`, but it’s there as a safety precaution.
+    let lev_ratio = (lev_distance as f32 / word_len as f32).min(1.);
+
+    1. - lev_ratio
+}
+
+#[cfg(test)]
+#[test]
+fn test_typo_score() {
+    // No typo.
+    assert_eq!(typo_score(0, 1), 1.);
+    assert_eq!(typo_score(0, 2), 1.);
+
+    // 1 typo.
+    assert_eq!(typo_score(1, 3), 0.6666666);
+    assert_eq!(typo_score(1, 4), 0.75);
+    assert_eq!(typo_score(1, 5), 0.8);
+    // 1 typo always scores lower than 0.
+    for n in 2..=8 {
+        assert!(typo_score(1, n) < typo_score(0, n), "n={n}");
+    }
+
+    // 2 typos.
+    assert_eq!(typo_score(2, 5), 0.6);
+    assert_eq!(typo_score(2, 6), 0.6666666);
+    assert_eq!(typo_score(2, 7), 0.71428573);
+    // 2 typos always scores lower than 1.
+    for n in 3..=8 {
+        assert!(typo_score(2, n) < typo_score(1, n), "n={n}");
+    }
+
+    // A lot of typos (length has no impact, proportion has).
+    assert_eq!(typo_score(1 * 20, 2 * 20), typo_score(1, 2));
+    assert_eq!(typo_score(3 * 20, 7 * 20), typo_score(3, 7));
+}
+
+fn overall_score(scores: &[Option<QueryMatchScore>]) -> QueryResultScore {
+    let total = scores.iter().map(|opt| opt.unwrap_or(0f32)).sum::<f32>();
+    let count = scores.len() as f32;
+
+    #[allow(clippy::let_and_return)]
+    let average = total / count;
+
+    average
+}
+
+#[cfg(test)]
+#[test]
+fn test_overall_score() {
+    const MISSING: Option<QueryMatchScore> = None;
+    const EXACT_MATCH: Option<QueryMatchScore> = Some(1.);
+
+    // Max score for exact matches.
+    assert_eq!(overall_score(&[EXACT_MATCH; 1]), 1.);
+    assert_eq!(overall_score(&[EXACT_MATCH; 2]), 1.);
+    assert_eq!(overall_score(&[EXACT_MATCH; 3]), 1.);
+    assert_eq!(overall_score(&[EXACT_MATCH; 4]), 1.);
+
+    // Lowest score for missing matches.
+    assert_eq!(overall_score(&[MISSING; 1]), 0.);
+    assert_eq!(overall_score(&[MISSING; 2]), 0.);
+    assert_eq!(overall_score(&[MISSING; 3]), 0.);
+    assert_eq!(overall_score(&[MISSING; 4]), 0.);
+
+    // Auto-complete > fuzzy matching (not always, but in most cases).
+    assert!(overall_score(&[Some(prefix_score(5, 4))]) > overall_score(&[Some(typo_score(1, 10))]));
+
+    // Missing one term.
+    assert_eq!(overall_score(&[MISSING, EXACT_MATCH]), 1. / 2.);
+    assert_eq!(overall_score(&[MISSING, EXACT_MATCH, EXACT_MATCH]), 2. / 3.);
+    // Missing one term is better than missing all terms.
+    assert!(overall_score(&[MISSING, EXACT_MATCH]) > overall_score(&[MISSING]));
+
+    // Term order has no meaning.
+    assert_eq!(
+        overall_score(&[
+            EXACT_MATCH,
+            Some(prefix_score(2, 3)),
+            Some(typo_score(2, 7))
+        ]),
+        overall_score(&[
+            Some(typo_score(2, 7)),
+            Some(prefix_score(2, 3)),
+            EXACT_MATCH
+        ])
+    );
+
+    // All typos in one term is like the same total across multiple terms.
+    // NOTE: This is not a requirement, it’s just a non-regression test.
+    assert_eq!(
+        overall_score(&[Some(typo_score(1, 7)); 2]),
+        overall_score(&[Some(typo_score(2, 7)), EXACT_MATCH])
+    );
+    assert_eq!(
+        overall_score(&[Some(typo_score(1, 7)); 3]),
+        overall_score(&[Some(typo_score(3, 7)), EXACT_MATCH, EXACT_MATCH])
+    );
+
+    // Examples for “The brown fox jumps over the lazy dog”:
+    // “brown fox jumps”
+    assert_eq!(overall_score(&[EXACT_MATCH, EXACT_MATCH, EXACT_MATCH]), 1.);
+    // “brown fox jum”
+    assert_eq!(
+        overall_score(&[EXACT_MATCH, EXACT_MATCH, Some(prefix_score(2, 3))]),
+        0.9892473
+    );
+    // “bron fox jum”
+    assert_eq!(
+        overall_score(&[
+            Some(typo_score(1, 5)),
+            EXACT_MATCH,
+            Some(prefix_score(2, 3))
+        ]),
+        0.92258066
+    );
+    // “brown fox”
+    assert_eq!(overall_score(&[EXACT_MATCH, EXACT_MATCH,]), 1.);
+    // “brown fox eats”
+    assert_eq!(
+        overall_score(&[EXACT_MATCH, EXACT_MATCH, MISSING]),
+        0.6666667
+    ); // 2/3
+}
+
+fn document_frequency(term_hash: StoreTermHash, kv_action: &KvStoreActionReadOnly<'_>) -> u64 {
+    kv_action
+        .get_term_to_iids(term_hash)
+        .inspect_err(|err| tracing::error!("{err:?}"))
+        .unwrap_or(None)
+        .map_or(0, |iids| iids.len()) as u64
+}
+
+fn bm25_lite_idf(document_count: u64, document_frequency: u64) -> f32 {
+    debug_assert!(
+        document_frequency <= document_count,
+        "{document_frequency} > {document_count}"
+    );
+
+    let document_count = document_count.max(document_frequency) as f64;
+    let df = document_frequency as f64;
+
+    (1.0 + (document_count - df + 0.5) / (df + 0.5)).ln() as f32
+}
+
+#[allow(clippy::too_many_arguments)] // We’ll refactor this someday, and it’ not public anyway.
+fn merge_suggestions(
+    suggestions: impl Iterator<Item = (String, QueryMatchScore)>,
+    scoring_matrix: &mut IndexMap<StoreObjectIid, Vec<Option<QueryMatchScore>>>,
+    term: &str,
+    term_idx: usize,
+    term_count: usize,
+    kv_action: &KvStoreActionReadOnly<'_>,
+    alternates_try: &mut usize,
+    higher_limit: usize,
+    document_count: u64,
+    minimum_idf: f32,
+) {
+    'suggestions: for (suggested_word, base_score) in suggestions {
+        // Do not load base results twice for same term as base term
+        if suggested_word.eq(term) {
+            continue;
+        }
+
+        tracing::trace!(?term, ?suggested_word, "got completed word for term");
+
+        let suggested_term_hash = StoreTermHash::from(suggested_word.as_str());
+        let suggested_iids = match kv_action.get_term_to_iids(suggested_term_hash) {
+            Ok(Some(suggested_iids)) => suggested_iids,
+            Ok(None) => continue,
+            Err(_) => continue,
+        };
+
+        let document_frequency = document_frequency(suggested_term_hash, kv_action);
+
+        // Filter out minimum IDF.
+        // PERF: Filtering `minimum_idf > 0` to save some computation.
+        if minimum_idf > 0. {
+            let idf = (document_count as f32 / document_frequency as f32).ln();
+            if idf < minimum_idf {
+                tracing::debug!(
+                    "skipping term {suggested_word:?} because idf too low ({idf}<{minimum_idf})"
+                );
+                continue;
+            }
+        }
+
+        let bm25_score = bm25_lite_idf(document_count, document_frequency);
+
+        let suggestion_score = base_score * bm25_score;
+
+        for suggested_iid in suggested_iids.into_iter().take(*alternates_try) {
+            // SAFETY: We can reach at most `alternates_try`.
+            *alternates_try = unsafe { alternates_try.unchecked_sub(1) };
+
+            let inserted = update_score(
+                scoring_matrix,
+                suggested_iid,
+                suggestion_score,
+                term_idx,
+                term_count,
+            );
+
+            if inserted {
+                // Higher limit now reached?
+                // Stop acquiring new suggested IIDs now.
+                if scoring_matrix.len() >= higher_limit {
+                    tracing::trace!(?term, "got enough completed results for term");
+
+                    break 'suggestions;
+                }
+            }
+        }
+    }
+
+    tracing::trace!(
+        ?term,
+        "done completing results for term, now {} total results",
+        scoring_matrix.len()
+    );
+}
+
+fn update_score(
+    scoring_matrix: &mut IndexMap<StoreObjectIid, Vec<Option<QueryMatchScore>>>,
+    iid: StoreObjectIid,
+    score: QueryMatchScore,
+    term_idx: usize,
+    term_count: usize,
+) -> bool {
+    match scoring_matrix.entry(iid) {
+        // If entry already exists, use lowest score.
+        indexmap::map::Entry::Occupied(mut occupied_entry) => {
+            // SAFETY: We always initialize vecs with `term_count` entries.
+            let entry_score = unsafe { occupied_entry.get_mut().get_unchecked_mut(term_idx) };
+
+            let new_score = entry_score.map_or(score, |entry_score| score.min(entry_score));
+
+            tracing::trace!(entry_score, new_score, "Updating to min score");
+            *entry_score = Some(new_score);
+
+            false
+        }
+        // If entry does not exist, insert score.
+        indexmap::map::Entry::Vacant(vacant_entry) => {
+            let mut scores = vec![None; term_count];
+
+            tracing::trace!(new_score = score, "Inserting new score");
+            // SAFETY: `scores` has `term_count` elements.
+            unsafe { *scores.get_unchecked_mut(term_idx) = Some(score) };
+
+            vacant_entry.insert(scores);
+
+            true
+        }
+    }
+}

core/src/executor/suggest.rs

@@ -0,0 +1,42 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+use super::types::QuerySearchLimit;
+use crate::lexer::preprocessor::PreprocessorOutput;
+use crate::store::StoreItemPart;
+
+impl super::Executor {
+    pub fn suggest(
+        &self,
+        collection: StoreItemPart,
+        bucket: StoreItemPart,
+        input: PreprocessorOutput,
+        limit: QuerySearchLimit,
+    ) -> Result<Option<impl ExactSizeIterator<Item = String> + DoubleEndedIterator>, ()> {
+        // Important: acquire graph access read lock, and reference it in context. This \
+        //   prevents the graph from being erased while using it in this block.
+        let _fst_read_guard = self.fst_pool.lock_read_access();
+
+        if let Ok(fst_store) = self.fst_pool.acquire(collection, bucket) {
+            let mut tokens = input.tokens();
+
+            if let (Some(token), None) = (tokens.next(), tokens.next()) {
+                let len = token.as_original().len();
+                let term = token.into_normalized();
+
+                tracing::debug!("running suggest on word: {term:?}");
+
+                return match fst_store.suggest_words(term, len, limit as usize, None) {
+                    Some(words) => Ok(Some(words.map(|(k, _)| k))),
+                    None => Ok(None),
+                };
+            }
+        }
+
+        Err(())
+    }
+}

core/src/store/fst/mod.rs

@@ -0,0 +1,502 @@
+// Sonic
+//
+// Fast, lightweight and schema-less search backend
+// Copyright: 2019, Valerian Saliou <valerian@valeriansaliou.name>
+// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
+// License: Mozilla Public License v2.0 (MPL v2.0)
+
+mod backup;
+mod pool;
+mod util;
+
+use std::fmt;
+use std::sync::{Arc, RwLock};
+use std::time::SystemTime;
+
+use fst::{IntoStreamer as _, Streamer as _};
+use hashbrown::HashSet;
+use regex_syntax::escape as regex_escape;
+
+use crate::lexer::ranges::LexerRegexRange;
+
+use super::generic::*;
+
+pub use self::pool::{FstStoreId, FstStorePool};
+use self::util::*;
+
+pub struct FstStore {
+    graph: fst::Set,
+    target: FstStoreId,
+    pending: FstStorePending,
+    last_used: Arc<RwLock<SystemTime>>,
+    last_consolidated: Arc<RwLock<SystemTime>>,
+    graph_consolidate: Arc<RwLock<HashSet<FstStoreId>>>,
+    // NOTE: This shouldn’t be here, but until a big rewrite let’s not care.
+    action_config: FstStoreActionConfig,
+}
+
+#[derive(Default)]
+pub struct FstStorePending {
+    pop: Arc<RwLock<HashSet<Vec<u8>>>>,
+    push: Arc<RwLock<HashSet<Vec<u8>>>>,
+}
+
+pub struct FstStoreActionBuilder<'build> {
+    pub fst_store_config: &'build crate::config::FstStoreConfig,
+}
+
+type FstStoreAtom = u32;
+
+pub struct FstStoreMisc;
+
+#[derive(Copy, Clone)]
+enum FstStorePathMode {
+    Permanent,
+    Temporary,
+    Backup,
+}
+
+impl FstStorePathMode {
+    fn extension(&self) -> &'static str {
+        match self {
+            FstStorePathMode::Permanent => ".fst",
+            FstStorePathMode::Temporary => ".fst.tmp",
+            FstStorePathMode::Backup => ".fst.bck",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct FstStoreActionConfig {
+    pub prefix_matching_enabled: bool,
+    pub fuzzy_matching_enabled: bool,
+}
+
+impl Default for FstStoreActionConfig {
+    fn default() -> Self {
+        Self {
+            prefix_matching_enabled: true,
+            fuzzy_matching_enabled: true,
+        }
+    }
+}
+
+const WORD_LIMIT_LENGTH: usize = 40;
+
+impl FstStore {
+    pub fn cardinality(&self) -> usize {
+        self.graph.len()
+    }
+
+    pub fn as_stream(&self) -> fst::set::Stream<'_> {
+        self.graph.into_stream()
+    }
+
+    pub fn lookup_begins_(&self, word: &str) -> Result<fst::set::Stream<'_, fst_regex::Regex>, ()> {
+        // NOTE: This regex maps over an unicode range, for speed reasons at scale.
+        //   We found out that the 'match any' syntax ('.*') was super-slow. Using the restrictive
+        //   syntax below divided the cost of e.g. a search query by 2. The regex below has been
+        //   found out to be nearly zero-cost to compile and execute, for whatever reason.
+        // Regex format: '{escaped_word}([{unicode_range}]*)'
+        let mut regex_str = regex_escape(word);
+
+        regex_str.push('(');
+
+        LexerRegexRange::from(word)
+            .unwrap_or_default()
+            .write_to(&mut regex_str)
+            // Regex write failed? (this should not happen)
+            .map_err(|error| tracing::error!(
+                "Could not lookup word in fst via 'begins': {word:?} because regex write failed: {error:?}"
+            ))?;
+
+        regex_str.push_str("*)");
+
+        // Proceed word lookup.
+        tracing::debug!("Looking-up word in fst via 'begins': {word:?} with regex: {regex_str:?}");
+
+        let regex = fst_regex::Regex::new(&regex_str).map_err(|_error| ())?;
+
+        Ok(self.graph.search(regex).into_stream())
+    }
+
+    pub fn lookup_typos_(
+        &self,
+        word: &str,
+        typo_factor: u32,
+    ) -> Result<fst::set::Stream<'_, fst_levenshtein::Levenshtein>, ()> {
+        tracing::debug!(
+            "Looking-up word in fst via 'typos': {word:?} with typo factor: {typo_factor:?}"
+        );
+
+        let fuzzy = fst_levenshtein::Levenshtein::new(word, typo_factor).map_err(|_error| ())?;
+
+        Ok(self.graph.search(fuzzy).into_stream())
+    }
+
+    pub fn should_consolidate(&self) {
+        let id = self.target;
+
+        // Check if not already scheduled.
+        if self.graph_consolidate.read().unwrap().contains(&id) {
+            tracing::debug!("Graph consolidation already scheduled on pool: {id}");
+            return;
+        };
+
+        // Schedule target for next consolidation tick (i.e. collection + bucket tuple).
+        self.graph_consolidate.write().unwrap().insert(id);
+
+        // Bump “last consolidated” time, effectively de-bouncing consolidation
+        // to a fixed and predictable tick time in the future.
+        let mut last_consolidated_value = self.last_consolidated.write().unwrap();
+
+        *last_consolidated_value = SystemTime::now();
+
+        // Perform an early drop of the lock (frees up write lock early).
+        drop(last_consolidated_value);
+
+        tracing::info!("Graph consolidation scheduled on pool: {id}");
+    }
+}
+
+impl StoreGeneric for FstStore {
+    fn ref_last_used(&self) -> &RwLock<SystemTime> {
+        &self.last_used
+    }
+}
+
+impl FstStore {
+    pub fn push_word(&self, word: &str, fst_store_config: &crate::config::FstStoreConfig) -> bool {
+        // Word over limit? (abort, the FST does not perform well over large words)
+        if Self::word_over_limit(word) {
+            return false;
+        }
+
+        let word_bytes = word.as_bytes();
+
+        // Nuke word from 'pop' set? (void a previous un-consolidated commit)
+        if self.pending.pop.read().unwrap().contains(word_bytes) {
+            self.pending.pop.write().unwrap().remove(word_bytes);
+        }
+
+        // Add word in 'push' set? (only if word is not in FST)
+        // NOTE: also check whether FST is over limits or not from there, to avoid
+        //   stacking words that could never be consolidated to final FST anyway.
+        let graph_fst = self.graph.as_fst();
+
+        if self.graph.contains(&word) {
+            return false;
+        }
+
+        if check_over_limits(graph_fst.size(), graph_fst.len(), &fst_store_config.graph) {
+            return false;
+        }
+
+        {
+            let pending_push_guard = self.pending.push.read().unwrap();
+
+            if pending_push_guard.contains(word_bytes)
+                || pending_push_guard.len() >= fst_store_config.graph.max_words
+            {
+                return false;
+            }
+        }
+
+        (self.pending.push.write().unwrap()).insert(word_bytes.to_vec());
+
+        self.should_consolidate();
+
+        true
+    }
+
+    pub fn pop_word(&self, word: &str) -> bool {
+        // Word over limit? (abort, the FST does not perform well over large words)
+        if Self::word_over_limit(word) {
+            return false;
+        }
+
+        let word_bytes = word.as_bytes();
+
+        // Nuke word from 'push' set? (void a previous un-consolidated commit)
+        if self.pending.push.read().unwrap().contains(word_bytes) {
+            self.pending.push.write().unwrap().remove(word_bytes);
+        }
+
+        if !self.graph.contains(word_bytes) {
+            return false;
+        }
+
+        // Add word in 'pop' set? (only if word is in FST)
+        if self.pending.pop.read().unwrap().contains(word_bytes) {
+            return false;
+        }
+
+        (self.pending.pop.write().unwrap()).insert(word_bytes.to_vec());
+
+        self.should_consolidate();
+
+        true
+    }
+
+    pub fn suggest_words(
+        &self,
+        from_word: &str,
+        // Length before stemming. Useful to apply fuzzy matching rules based
+        // on user input.
+        original_word_len: usize,
+        limit: usize,
+        max_typo_factor: Option<u32>,
+    ) -> Option<impl ExactSizeIterator<Item = (String, u16)> + DoubleEndedIterator + use<>> {
+        use indexmap::IndexMap;
+
+        // Word over limit? (abort, the FST does not perform well over large words)
+        if Self::word_over_limit(from_word) {
+            return None;
+        }
+
+        let mut found_words: IndexMap<String, u16> = IndexMap::with_capacity(limit);
+
+        if self.action_config.prefix_matching_enabled {
+            // Try to complete provided word
+            if let Some(stream) = self.lookup_begins(from_word, original_word_len) {
+                for (word, score) in stream {
+                    if found_words.contains_key(&word) {
+                        continue;
+                    }
+
+                    found_words.insert(word, score);
+
+                    // Requested limit reached? Stop there.
+                    if found_words.len() >= limit {
+                        break;
+                    }
+                }
+            }
+        }
+
+        // Try to fuzzy-suggest other words? (e.g. correct typos)
+        if self.action_config.fuzzy_matching_enabled && found_words.len() < limit {
+            // Allow more typos in word as the word gets longer, up to a maximum limit
+            let max_typo_factor = max_typo_factor.unwrap_or(typo_factor(original_word_len));
+            let mut typo_factor = 1u32;
+
+            // TODO: Rework the Levenshtein query feature to avoid repeating
+            //   the same query over and over again. Maybe try to see if
+            //   `fst_levenshtein` can return distances in its response.
+            while found_words.len() < limit && typo_factor <= max_typo_factor {
+                let Some(stream) = self.lookup_typos(from_word, typo_factor) else {
+                    break;
+                };
+
+                for (word, score) in stream {
+                    if found_words.contains_key(&word) {
+                        continue;
+                    }
+
+                    found_words.insert(word, score);
+
+                    // Requested limit reached? Stop there.
+                    if found_words.len() >= limit {
+                        break;
+                    }
+                }
+
+                typo_factor += 1;
+            }
+        }
+
+        if !found_words.is_empty() {
+            Some(found_words.into_iter())
+        } else {
+            None
+        }
+    }
+
+    pub fn lookup_begins(
+        &self,
+        word: &str,
+        // Length before stemming. Useful to calculate correct score.
+        original_word_len: usize,
+    ) -> Option<impl Iterator<Item = (String, u16)>> {
+        // Word over limit? (abort, the FST does not perform well over large words)
+        if Self::word_over_limit(word) {
+            return None;
+        }
+
+        if !self.action_config.prefix_matching_enabled {
+            return None;
+        }
+
+        let Ok(stream) = self.lookup_begins_(word) else {
+            return None;
+        };
+
+        tracing::debug!(?word, "looking up for word in 'begins' fst stream");
+
+        Some(FstStreamIterator(stream).map(move |word| {
+            // WARN: Calculating distance to original word length might
+            //   yield weird results when combines with stemming.
+            let distance: usize = original_word_len.abs_diff(word.len());
+            let score = u16::try_from(distance).unwrap_or(u16::MAX);
+            (word, score)
+        }))
+    }
+
+    pub fn lookup_typos(
+        &self,
+        word: &str,
+        typo_factor: u32,
+    ) -> Option<impl Iterator<Item = (String, u16)>> {
+        if !self.action_config.fuzzy_matching_enabled {
+            return None;
+        }
+
+        let Ok(stream) = self.lookup_typos_(word, typo_factor) else {
+            return None;
+        };
+
+        tracing::debug!(
+            ?word,
+            typo_factor,
+            "looking up for word in 'typos' fst stream"
+        );
+
+        // NOTE: Returning the same score for every word works only
+        //   because we re-run `lookup_typos` for increasingly
+        //   larger typo factors and do not re-insert existing
+        //   values. As explained in previous TODO, we should try
+        //   to get the real distance back from `fst_levenshtein`.
+        let score = u16::try_from(typo_factor).unwrap_or(u16::MAX);
+
+        Some(FstStreamIterator(stream).map(move |word| (word, score)))
+    }
+
+    pub fn list_words(&self, limit: usize, offset: usize) -> Result<Vec<String>, ()> {
+        let stream = self.as_stream();
+
+        // Enumerate words from FST stream.
+        match stream
+            .into_strs()
+            .map(|words| words.into_iter().skip(offset).take(limit).collect())
+        {
+            Err(err) => {
+                tracing::debug!("conversion of stream failed: {err:?}");
+                Err(())
+            }
+            Ok(words) => Ok(words),
+        }
+    }
+
+    pub fn count_words(&self) -> usize {
+        self.cardinality()
+    }
+
+    fn word_over_limit(word: &str) -> bool {
+        if word.len() > WORD_LIMIT_LENGTH {
+            tracing::debug!("got over-limit fst word: {word:?}");
+
+            true
+        } else {
+            false
+        }
+    }
+}
+
+/// Allow more typos in word as the word gets longer, up to a maximum limit.
+pub(crate) fn typo_factor(word_len: usize) -> u32 {
+    match word_len {
+        1..=3 => 0,
+        4..=6 => 1,
+        7..=9 => 2,
+        _ => 3,
+    }
+}
+
+// MARK: - Helpers
+
+#[repr(transparent)]
+struct FstStreamIterator<'a, A: fst::Automaton>(fst::set::Stream<'a, A>);
+
+impl<'a, A: fst::Automaton> Iterator for FstStreamIterator<'a, A> {
+    type Item = String;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        match self.0.next() {
+            Some(bytes) => match str::from_utf8(bytes) {
+                Ok(str) => Some(str.to_owned()),
+                Err(_) => None,
+            },
+            None => None,
+        }
+    }
+}
+
+// MARK: - Tests
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    pub(in crate::store::fst) fn test_fst_pool() -> FstStorePool {
+        let fst_store_config = test_fst_store_config();
+
+        FstStorePool::new(fst_store_config, Default::default())
+    }
+
+    pub(in crate::store::fst) fn test_fst_store_config() -> Arc<crate::config::FstStoreConfig> {
+        Arc::new(
+            config::Config::builder()
+                .add_source(config::File::from_str(
+                    crate::config::tests::defaults_toml(),
+                    config::FileFormat::Toml,
+                ))
+                .build()
+                .unwrap()
+                .get::<crate::config::FstStoreConfig>("store.fst")
+                .unwrap(),
+        )
+    }
+}
+
+// MARK: - Boilerplate
+
+impl fmt::Debug for FstStore {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use crate::util::fmt::AsPrettyRwLock;
+
+        // NOTE: Deconstructing to future-proof this function.
+        let Self {
+            graph,
+            target,
+            pending,
+            last_used,
+            last_consolidated,
+            graph_consolidate,
+            action_config,
+        } = self;
+
+        f.debug_struct("FstStore")
+            .field("graph", graph)
+            .field("target", target)
+            .field("pending", pending)
+            .field("last_used", &AsPrettyRwLock(last_used))
+            .field("last_consolidated", &AsPrettyRwLock(last_consolidated))
+            .field("graph_consolidate", &AsPrettyRwLock(graph_consolidate))
+            .field("action_config", action_config)
+            .finish()
+    }
+}
+
+impl fmt::Debug for FstStorePending {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use crate::util::fmt::AsPrettyRwLock;
+
+        // NOTE: Deconstructing to future-proof this function.
+        let Self { pop, push } = self;
+
+        f.debug_struct("FstStorePending")
+            .field("pop", &AsPrettyRwLock(pop))
+            .field("push", &AsPrettyRwLock(push))
+            .finish()
+    }
+}