diff --git a/pagefind/src/fossick/splitting.rs b/pagefind/src/fossick/splitting.rs index 35348ca..4ad6184 100644 --- a/pagefind/src/fossick/splitting.rs +++ b/pagefind/src/fossick/splitting.rs @@ -139,7 +139,7 @@ fn get_discrete_words>(s: S) -> (String, Option>) { let words = s .as_ref() - .replace(|c: char| c.is_ascii_punctuation(), " ") + .replace(|c: char| c.is_ascii_punctuation() || c == '\u{AD}', " ") .to_case(Case::Lower) .split_whitespace() .collect::>() @@ -210,10 +210,13 @@ mod tests { let camel_results = get_indexable_words("ThreeAntelopes", stemmer.as_ref(), &[]); let kebab_results = get_indexable_words("three-antelopes", stemmer.as_ref(), &[]); let snake_results = get_indexable_words("three_antelopes", stemmer.as_ref(), &[]); + let soft_hyphen_results = + get_indexable_words("three\u{AD}antelopes", stemmer.as_ref(), &[]); assert_eq!(camel_results, expected); assert_eq!(kebab_results, expected); assert_eq!(snake_results, expected); + assert_eq!(soft_hyphen_results, expected); } #[test] @@ -258,6 +261,15 @@ mod tests { assert_eq!(get_discrete_words(input), ("array structures".into(), None)); } + #[test] + fn soft_hyphenated_words() { + let input = "Hochspannungs\u{AD}netz\u{AD}teil"; + assert_eq!( + get_discrete_words(input), + ("hochspannungs netz teil".into(), None) + ); + } + #[test] fn camel_words() { let input = "WKWebVIEWComponent";