From adeab236650e1a283e130bedc7d4f3f87da240e2 Mon Sep 17 00:00:00 2001 From: upstream Date: Fri, 2 Oct 2026 16:24:37 +0200 Subject: [PATCH] Upstream Pagefind/pagefind state of the touched files before 2e9704cb17 --- pagefind/src/fossick/splitting.rs | 317 ++++++++++++++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 pagefind/src/fossick/splitting.rs diff --git a/pagefind/src/fossick/splitting.rs b/pagefind/src/fossick/splitting.rs new file mode 100644 index 0000000..35348ca --- /dev/null +++ b/pagefind/src/fossick/splitting.rs @@ -0,0 +1,317 @@ +use convert_case::{Case, Casing}; +use emojis; +use lazy_static::lazy_static; +use pagefind_stem::Stemmer; +use regex::Regex; +use unicode_normalization::UnicodeNormalization; +use unicode_segmentation::UnicodeSegmentation; + +lazy_static! { + static ref EMOJI: Regex = Regex::new("\\p{Emoji}").unwrap(); +} + +#[derive(Debug, Clone, PartialEq)] +pub struct IndexableWord { + pub stemmed: String, + pub is_compound_part: bool, + /// The original word before diacritic normalization (if different) + pub original: Option, +} + +/// Returns all words that should be indexed for a given input word. +/// This includes: +/// - The primary normalized+stemmed word +/// - Compound word parts (for hyphenated/camelCase/etc words) +/// - Emoji characters +pub fn get_indexable_words( + raw_word: &str, + stemmer: Option<&Stemmer>, + include_characters: &[char], +) -> Vec { + let mut results = Vec::new(); + + let mut normalized_word = String::with_capacity(raw_word.len()); + for mut c in raw_word.chars() { + let is_alpha = c.is_alphanumeric(); + if is_alpha || include_characters.contains(&c) { + c.make_ascii_lowercase(); + if c.is_uppercase() { + // Non-ascii uppercase can lower to multiple chars + normalized_word.extend(c.to_lowercase()); + } else { + normalized_word.push(c); + } + } + } + + if !normalized_word.is_empty() { + let diacritic_normalized = normalize_diacritics(&normalized_word); + + let stemmed = if let Some(stemmer) = stemmer { + stemmer + .stem(diacritic_normalized.as_deref().unwrap_or(&normalized_word)) + .into_owned() + } else { + diacritic_normalized + .clone() + .unwrap_or_else(|| normalized_word.to_string()) + }; + + results.push(IndexableWord { + stemmed, + original: diacritic_normalized.map(|_| normalized_word.to_string()), + is_compound_part: false, + }); + } + + let possibly_compound = raw_word.chars().any(|c| !c.is_alphanumeric()) + || raw_word.chars().skip(1).any(|c| c.is_uppercase()); + + if !possibly_compound { + return results; + } + + let (word_parts, extras) = get_discrete_words(raw_word); + if !normalized_word.is_empty() + && (word_parts.contains(|c: char| c.is_whitespace()) + || !normalized_word.starts_with(&word_parts)) + { + let part_words: Vec<_> = word_parts.split_whitespace().collect(); + + // Only index two+ character words + for part_word in part_words.into_iter().filter(|w| w.len() > 1) { + let part_diacritic = normalize_diacritics(part_word); + + let stemmed_part = if let Some(stemmer) = stemmer { + stemmer + .stem(part_diacritic.as_deref().unwrap_or(part_word)) + .into_owned() + } else { + part_diacritic + .clone() + .unwrap_or_else(|| part_word.to_string()) + }; + + results.push(IndexableWord { + stemmed: stemmed_part, + original: part_diacritic.map(|_| part_word.to_string()), + is_compound_part: true, + }); + } + } + + if let Some(extras) = extras { + for extra in extras { + results.push(IndexableWord { + stemmed: extra, + original: None, + is_compound_part: false, + }); + } + } + + results +} + +/// Normalize diacritics, if required, before indexing +/// (context: we index "café" as "cafe" for retrieval purposes, +/// but we do still store each variant in the index, +/// so matching diacritics are preferred when ranking search results) +fn normalize_diacritics(word: &str) -> Option { + if word.is_ascii() { + return None; + } + + let normalized: String = word + .nfd() + .filter(|c| !unicode_normalization::char::is_combining_mark(*c)) + .collect(); + + if normalized != word { + Some(normalized) + } else { + None + } +} + +fn get_discrete_words>(s: S) -> (String, Option>) { + let mut extras = None; + + let words = s + .as_ref() + .replace(|c: char| c.is_ascii_punctuation(), " ") + .to_case(Case::Lower) + .split_whitespace() + .collect::>() + .join(" "); + + if EMOJI.is_match(s.as_ref()) { + extras = Some( + s.as_ref() + .graphemes(true) + .into_iter() + .filter_map(|x| { + if emojis::get(x).is_some() { + Some(x.to_string()) + } else { + None + } + }) + .collect::>(), + ); + } + + (words, extras) +} + +#[cfg(test)] +mod tests { + use crate::fossick::get_stemmer; + + use super::*; + + #[test] + fn indexable_words_simple() { + let stemmer = get_stemmer("en"); + let results = get_indexable_words("hello", stemmer.as_ref(), &[]); + + assert_eq!( + results, + vec![IndexableWord { + stemmed: "hello".to_string(), + is_compound_part: false, + original: None + }] + ); + } + + #[test] + fn indexable_words_compound() { + let stemmer = get_stemmer("en"); + + let expected = vec![ + IndexableWord { + stemmed: "threeantelop".to_string(), + is_compound_part: false, + original: None, + }, + IndexableWord { + stemmed: "three".to_string(), + is_compound_part: true, + original: None, + }, + IndexableWord { + stemmed: "antelop".to_string(), + is_compound_part: true, + original: None, + }, + ]; + + let camel_results = get_indexable_words("ThreeAntelopes", stemmer.as_ref(), &[]); + let kebab_results = get_indexable_words("three-antelopes", stemmer.as_ref(), &[]); + let snake_results = get_indexable_words("three_antelopes", stemmer.as_ref(), &[]); + + assert_eq!(camel_results, expected); + assert_eq!(kebab_results, expected); + assert_eq!(snake_results, expected); + } + + #[test] + fn indexable_words_compound_diacritics() { + let stemmer = get_stemmer("fr"); + let results = get_indexable_words("café-crème", stemmer.as_ref(), &[]); + + assert_eq!( + results, + vec![ + IndexableWord { + stemmed: "cafecrem".to_string(), + is_compound_part: false, + original: Some("cafécrème".to_string()), + }, + IndexableWord { + stemmed: "caf".to_string(), + is_compound_part: true, + original: Some("café".to_string()), + }, + IndexableWord { + stemmed: "crem".to_string(), + is_compound_part: true, + original: Some("crème".to_string()), + }, + ] + ); + } + + #[test] + fn hyphenated_words() { + let input = "these-words-are-hyphenated"; + assert_eq!( + get_discrete_words(input), + ("these words are hyphenated".into(), None) + ); + } + + #[test] + fn underscored_words() { + let input = "__array_structures"; + assert_eq!(get_discrete_words(input), ("array structures".into(), None)); + } + + #[test] + fn camel_words() { + let input = "WKWebVIEWComponent"; + assert_eq!( + get_discrete_words(input), + ("wk web view component".into(), None) + ); + } + + #[test] + fn dotted_words() { + let input = "page.Find"; + assert_eq!(get_discrete_words(input), ("page find".into(), None)); + } + + #[test] + fn misc_punctuation() { + let input = "cloud/cannon,page.find"; + assert_eq!( + get_discrete_words(input), + ("cloud cannon page find".into(), None) + ); + } + + #[test] + fn french() { + let input = "l'alphabet"; + assert_eq!(get_discrete_words(input), ("l alphabet".into(), None)); + } + + #[test] + fn html() { + let input = ""; + assert_eq!( + get_discrete_words(input), + ("form component data pagefind meta key value".into(), None) + ); + } + + #[test] + fn emoji() { + let input = "cloud🌦️cannon"; + assert_eq!( + get_discrete_words(input), + ("cloud🌦️cannon".into(), Some(vec!["🌦️".into()])) + ); + + let input = "👋👨‍👩‍👧‍👦🌾"; + assert_eq!( + get_discrete_words(input), + ( + "👋👨‍👩‍👧‍👦🌾".into(), + Some(vec!["👋".into(), "👨‍👩‍👧‍👦".into(), "🌾".into()]) + ) + ); + } +}