Upstream valeriansaliou/sonic state of the touched files before abd3d56935
This commit is contained in:
upstream committed
1 parent
73692bcfdb
commit
0f79d0dab8
11 files changed
+2450
-1
No files matched your search
@@ -0,0 +1,80 @@
|
||||
// Sonic
|
||||
//
|
||||
// Fast, lightweight and schema-less search backend
|
||||
// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
|
||||
// License: Mozilla Public License v2.0 (MPL v2.0)
|
||||
|
||||
use crate::common::util::unique_hex;
|
||||
|
||||
pub fn make_config(toml: &str) -> sonic::Config {
|
||||
let raw_config: config::Config = config::Config::builder()
|
||||
.add_source(config::File::from_str(toml, config::FileFormat::Toml))
|
||||
.build()
|
||||
.expect("error reading config");
|
||||
|
||||
// Parse configuration.
|
||||
let config = raw_config
|
||||
.try_deserialize::<sonic::Config>()
|
||||
.expect("syntax error in config");
|
||||
|
||||
// Validate configuration.
|
||||
config.validate();
|
||||
|
||||
config
|
||||
}
|
||||
|
||||
pub fn defaults_toml() -> String {
|
||||
let test_id = unique_hex().unwrap();
|
||||
|
||||
let test_data_path = std::path::Path::new(env!("CARGO_TARGET_TMPDIR")).join(&test_id);
|
||||
std::fs::create_dir_all(&test_data_path).unwrap();
|
||||
|
||||
let kv_store_path = test_data_path.join("kv-store");
|
||||
std::fs::create_dir(&kv_store_path).unwrap();
|
||||
let fst_store_path = test_data_path.join("fst-store");
|
||||
std::fs::create_dir(&fst_store_path).unwrap();
|
||||
|
||||
format!(
|
||||
r#"
|
||||
[normalization]
|
||||
unicode_normalization = "none"
|
||||
diacritic_folding_enabled = true
|
||||
stemming_enabled = false
|
||||
|
||||
[tokenization]
|
||||
detect_special_patterns = true
|
||||
compat_split_special_patterns = false
|
||||
|
||||
[stopwords]
|
||||
allow = []
|
||||
deny = []
|
||||
|
||||
[search]
|
||||
query_limit_default = 10
|
||||
query_limit_maximum = 100
|
||||
query_alternates_try = 4
|
||||
query_minimum_term_idf_default = 0.1
|
||||
query_minimum_term_idf_minimum_object_count = 100
|
||||
suggest_limit_default = 5
|
||||
suggest_limit_maximum = 20
|
||||
list_limit_default = 100
|
||||
list_limit_maximum = 500
|
||||
|
||||
[store.kv]
|
||||
path = {kv_store_path:?}
|
||||
retain_word_objects = 1000
|
||||
pool.inactive_after = 1800
|
||||
database.flush_after = 900
|
||||
database.compression_type = "zstd"
|
||||
database.parallelism = 2
|
||||
database.write_ahead_log = true
|
||||
|
||||
[store.fst]
|
||||
path = {fst_store_path:?}
|
||||
pool.inactive_after = 300
|
||||
graph.consolidate_after = 180
|
||||
graph.max_size = 2048
|
||||
graph.max_words = 250000
|
||||
"#,
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Sonic
|
||||
//
|
||||
// Fast, lightweight and schema-less search backend
|
||||
// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
|
||||
// License: Mozilla Public License v2.0 (MPL v2.0)
|
||||
|
||||
#![allow(unused)]
|
||||
|
||||
pub mod config;
|
||||
mod executor;
|
||||
mod logging;
|
||||
mod meta_tests;
|
||||
pub mod util;
|
||||
|
||||
pub use self::executor::*;
|
||||
pub(crate) use self::item_ref::*;
|
||||
pub use self::logging::*;
|
||||
pub(crate) use self::meta_tests::*;
|
||||
pub(crate) use self::util::assert_contains;
|
||||
|
||||
// NOTE: Using macros instead of functions so `unwrap`s point to the call site
|
||||
// (helps debugging).
|
||||
pub(crate) mod item_ref {
|
||||
macro_rules! collection_ref {
|
||||
($collection:expr) => {
|
||||
sonic::store::StoreItemBuilder::from_depth_1($collection).unwrap()
|
||||
};
|
||||
}
|
||||
pub(crate) use collection_ref;
|
||||
|
||||
macro_rules! bucket_ref {
|
||||
($collection:expr, $bucket:expr) => {
|
||||
sonic::store::StoreItemBuilder::from_depth_2($collection, $bucket).unwrap()
|
||||
};
|
||||
}
|
||||
pub(crate) use bucket_ref;
|
||||
|
||||
macro_rules! object_ref {
|
||||
($collection:expr, $bucket:expr, $object:expr) => {
|
||||
sonic::store::StoreItemBuilder::from_depth_3($collection, $bucket, $object).unwrap()
|
||||
};
|
||||
}
|
||||
pub(crate) use object_ref;
|
||||
}
|
||||
|
||||
macro_rules! exec {
|
||||
($executor:ident -> PUSH $collection:tt $bucket:tt $oid:tt $text:tt $(LANG($lang:expr))? NEW) => {
|
||||
exec!(internal_ $executor -> PUSH $collection $bucket $oid $text true $(LANG($lang))?)
|
||||
};
|
||||
($executor:ident -> PUSH $collection:tt $bucket:tt $oid:tt $text:tt $(LANG($lang:expr))?) => {
|
||||
exec!(internal_ $executor -> PUSH $collection $bucket $oid $text false $(LANG($lang))?)
|
||||
};
|
||||
(internal_ $executor:ident -> PUSH $collection:tt $bucket:tt $oid:tt $text:tt $assume_new:ident $(LANG($lang:expr))?) => {{
|
||||
#[rustfmt::skip]
|
||||
$executor.log(format!(
|
||||
"PUSH {:?} {:?} {:?} {:?}{}{}",
|
||||
$collection, $bucket, $oid, $text, exec!(internal_ lang_txt $($lang)?), if $assume_new { " NEW" } else { "" }
|
||||
));
|
||||
let (c, b, o) = crate::common::object_ref!($collection, $bucket, $oid);
|
||||
let preprocessor = sonic::lexer::preprocessor::Preprocessor::new(
|
||||
$executor.app_conf.tokenization,
|
||||
$executor.app_conf.normalization,
|
||||
$executor.app_conf.stopwords.clone(),
|
||||
true,
|
||||
true,
|
||||
);
|
||||
$executor
|
||||
.push(
|
||||
c, b, o,
|
||||
preprocessor.preprocess($text, exec!(internal_ lang $($lang)?)),
|
||||
$assume_new,
|
||||
)
|
||||
.unwrap()
|
||||
}};
|
||||
|
||||
($executor:ident -> TRIGGER consolidate) => {{
|
||||
$executor.log(format!("TRIGGER consolidate"));
|
||||
$executor.fst_pool.consolidate(true, |_| true)
|
||||
}};
|
||||
|
||||
($executor:ident -> COUNT $collection:tt) => {{
|
||||
$executor.log(format!("COUNT {:?}", $collection));
|
||||
let c = collection_ref!($collection);
|
||||
$executor.countc(c)
|
||||
}};
|
||||
|
||||
($executor:ident -> COUNT $collection:tt $bucket:tt) => {{
|
||||
$executor.log(format!("COUNT {:?} {:?}", $collection, $bucket));
|
||||
let (c, b) = bucket_ref!($collection, $bucket);
|
||||
$executor.legacy_countb(c, b)
|
||||
}};
|
||||
|
||||
($executor:ident -> COUNTB $collection:tt $bucket:tt) => {{
|
||||
$executor.log(format!("COUNTB {:?} {:?}", $collection, $bucket));
|
||||
let (c, b) = bucket_ref!($collection, $bucket);
|
||||
$executor.countb(c, b)
|
||||
}};
|
||||
|
||||
($executor:ident -> COUNT $collection:tt $bucket:tt $oid:tt) => {{
|
||||
$executor.log(format!("COUNT {:?} {:?} {:?}", $collection, $bucket, $oid));
|
||||
let (c, b, o) = object_ref!($collection, $bucket, $oid);
|
||||
$executor.counto(c, b, o)
|
||||
}};
|
||||
|
||||
($executor:ident -> QUERY $collection:tt $bucket:tt $term:tt $(LANG($lang:expr))? $(LIMIT($limit:expr))?) => {{
|
||||
#[rustfmt::skip]
|
||||
$executor.log(format!(
|
||||
"QUERY {:?} {:?} {:?}{}{}",
|
||||
$collection, $bucket, $term, exec!(internal_ lang_txt $($lang)?), exec!(internal_ limit_txt $($limit)?)
|
||||
));
|
||||
let (c, b) = crate::common::bucket_ref!($collection, $bucket);
|
||||
let preprocessor = sonic::lexer::preprocessor::Preprocessor::new(
|
||||
$executor.app_conf.tokenization,
|
||||
$executor.app_conf.normalization,
|
||||
$executor.app_conf.stopwords.clone(),
|
||||
true,
|
||||
true,
|
||||
);
|
||||
$executor
|
||||
.search(
|
||||
c, b,
|
||||
preprocessor.preprocess($term, exec!(internal_ lang $($lang)?)),
|
||||
exec!(internal_ limit $($limit)?),
|
||||
0,
|
||||
)
|
||||
.expect("QUERY should succeed")
|
||||
}};
|
||||
|
||||
($executor:ident -> LIST $collection:tt $bucket:tt $(LIMIT($limit:expr))?) => {{
|
||||
$executor.log(format!("LIST {:?} {:?}", $collection, $bucket));
|
||||
let (c, b) = crate::common::bucket_ref!($collection, $bucket);
|
||||
$executor
|
||||
.list(
|
||||
c, b,
|
||||
exec!(internal_ limit $($limit)?),
|
||||
0,
|
||||
)
|
||||
.expect("LIST should succeed")
|
||||
}};
|
||||
|
||||
($executor:ident -> FLUSHC $collection:tt) => {{
|
||||
$executor.log(format!("FLUSHC {:?}", $collection));
|
||||
let c = crate::common::collection_ref!($collection);
|
||||
$executor.flushc(c).unwrap()
|
||||
}};
|
||||
|
||||
($executor:ident -> FLUSHB $collection:tt $bucket:tt) => {{
|
||||
$executor.log(format!("FLUSHB {:?} {:?}", $collection, $bucket));
|
||||
let (c, b) = crate::common::bucket_ref!($collection, $bucket);
|
||||
$executor.flushb(c, b).unwrap()
|
||||
}};
|
||||
|
||||
($executor:ident -> FLUSHO $collection:tt $bucket:tt $oid:tt) => {{
|
||||
$executor.log(format!("FLUSHO {:?} {:?} {:?}", $collection, $bucket, $oid));
|
||||
let (c, b, o) = crate::common::object_ref!($collection, $bucket, $oid);
|
||||
$executor.flusho(c, b, o).unwrap()
|
||||
}};
|
||||
|
||||
(internal_ lang) => { None };
|
||||
(internal_ lang $lang:expr) => { whatlang::Lang::from_code($lang) };
|
||||
|
||||
(internal_ lang_txt) => { "" };
|
||||
(internal_ lang_txt $lang:expr) => { format!(" LANG({})", $lang) };
|
||||
|
||||
(internal_ limit) => { sonic::executor::QuerySearchLimit::MAX };
|
||||
(internal_ limit $limit:expr) => { $limit };
|
||||
|
||||
(internal_ limit_txt) => { "" };
|
||||
(internal_ limit_txt $limit:expr) => { format!(" LLIMIT({})", $limit) };
|
||||
}
|
||||
pub(crate) use exec;
|
||||
@@ -0,0 +1,85 @@
|
||||
// Sonic
|
||||
//
|
||||
// Fast, lightweight and schema-less search backend
|
||||
// Copyright: 2026, Rémi Bardon <remi@remibardon.name>
|
||||
// License: Mozilla Public License v2.0 (MPL v2.0)
|
||||
|
||||
//! Feature: Typo correction
|
||||
//!
|
||||
//! A “typo factor” is chosen based on the token’s length. Those tests ensure
|
||||
//! typo correction works, and ensure no regression in the quality of the
|
||||
//! results.
|
||||
//! At the moment of this writing, the mapping function is the following:
|
||||
//!
|
||||
//! ```
|
||||
//! let mut typo_factor = match word.len() {
|
||||
//! 1..=3 => 0,
|
||||
//! 4..=6 => 1,
|
||||
//! 7..=9 => 2,
|
||||
//! _ => 3,
|
||||
//! };
|
||||
//! ```
|
||||
|
||||
mod common;
|
||||
|
||||
use crate::common::*;
|
||||
|
||||
/// This test sentence contains words that are 3–10 characters-long, while
|
||||
/// not being stopwords. It’s not realistic but not having stopwords avoids
|
||||
/// false positives in tests.
|
||||
const ASTRONOMY_WORDS: &str = "sun moon comet nebula pulsars asteroid satellite spacecraft";
|
||||
|
||||
/// Search should allow a certain number of typos (depending on token length).
|
||||
///
|
||||
/// This covers missing letters, inverted letters and additional letters.
|
||||
/// The underlying algorithm used is the Levenshtein distance, so this test is
|
||||
/// based on that knowledge.
|
||||
#[test]
|
||||
fn test_search_allows_typos() {
|
||||
// NOTE: Need to make language explicit because of
|
||||
// <https://github.com/valeriansaliou/sonic/issues/322#issuecomment-4638688602>.
|
||||
// Will be fixed separately.
|
||||
#[rustfmt::skip]
|
||||
test_ingest_then_query!(push: ASTRONOMY_WORDS [ensure_no_stopword] LANG("eng"), query: [
|
||||
// 3-letter word, distance = 1.
|
||||
("sum", false),
|
||||
// 4-letter word, distance = 1.
|
||||
("ssun", true), // sun
|
||||
("noon", true), // moon
|
||||
// 4-letter word, distance = 2.
|
||||
("commit", false),
|
||||
// 6-letter word, distance = 1.
|
||||
("nzbula", true), // nebula
|
||||
// 6-letter word, distance = 2.
|
||||
("nzbala", false),
|
||||
// 7-letter word, distance = 2.
|
||||
("plusars", true), // pulsars
|
||||
// 9-letter word, distance = 2.
|
||||
("saetllite", true), // satellite
|
||||
// 9-letter word, distance = 3.
|
||||
("satemmote", false),
|
||||
// 10-letter word, distance = 3.
|
||||
("sapcecrzft", true), // spacecraft
|
||||
// 10-letter word, distance = 4.
|
||||
("sapcecarft", false),
|
||||
] LANG("eng"));
|
||||
}
|
||||
|
||||
/// Ensures the order of words in search queries is insignificant.
|
||||
#[test]
|
||||
#[ignore = "Not supported yet (FIXME)"]
|
||||
fn test_search_term_order_insignificant() {
|
||||
#[rustfmt::skip]
|
||||
test_ingest_then_query!(push: ASTRONOMY_WORDS [ensure_no_stopword], query: [
|
||||
("satellite pulsars nebula", true),
|
||||
(&format!("missing {ASTRONOMY_WORDS}"), true),
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "Not supported yet"]
|
||||
fn test_chinese_typo_correction() {
|
||||
// We should make sure languages with shorter graphemes get typo correction
|
||||
// too.
|
||||
unimplemented!()
|
||||
}
|
||||
Reference in new issue
Block a user