fix(pagefind): skip exclude selectors that cannot be parsed

Every --exclude-selectors value was joined into a single selector string
handed to lol_html, so one selector the parser does not support took the
whole run down with an unwrap panic. Selectors are now validated one by
one and unsupported ones are reported through the logger and dropped.

Fixes #931
This commit is contained in:
dualfroz committed 2026-09-10 08:42:44 +02:00
1 parent c1c5f2ad73
commit 0428cf7195
1 file changed
+48 -2
+48 -2
View File
@@ -1,6 +1,6 @@
use lazy_static::lazy_static;
use lol_html::html_content::Element;
use lol_html::{element, text, HtmlRewriter, Settings};
use lol_html::{element, text, HtmlRewriter, Selector, Settings};
use regex::Regex;
use std::cell::RefCell;
use std::collections::BTreeMap;
@@ -138,7 +138,18 @@ impl<'a> DomParser<'a> {
custom_exclusions.extend(REMOVE_SELECTORS.iter().map(|s| s.to_string()));
let custom_exclusions = custom_exclusions
.iter()
.map(|e| format!("{} {}", options.root_selector, e))
.filter_map(|e| {
let selector = format!("{} {}", options.root_selector, e);
match selector.parse::<Selector>() {
Ok(_) => Some(selector),
Err(error) => {
options.logger.warn(format!(
"Ignoring the exclude selector {e:?}, which Pagefind can't parse: {error}. Indexing will continue without it."
));
None
}
}
})
.collect::<Vec<_>>()
.join(", ");
let mut anchor_counter = 0;
@@ -735,6 +746,41 @@ mod tests {
test_raw_parse(input)
}
fn test_parse_with_exclusions(
mut input: Vec<&'static str>,
exclusions: &[&'static str],
) -> DomParserResult {
use clap::CommandFactory;
let mut args = vec!["pagefind", "--source", "not_important"];
for exclusion in exclusions {
args.push("--exclude-selectors");
args.push(exclusion);
}
let config_args = vec![twelf::Layer::Clap(
crate::PagefindInboundConfig::command().get_matches_from(args),
)];
let config =
SearchOptions::load(crate::PagefindInboundConfig::with_layers(&config_args).unwrap())
.unwrap();
let mut rewriter = DomParser::new(&config);
input.insert(0, "<html><body>");
input.push("</body></html>");
for line in input {
let _ = rewriter.write(line.as_bytes());
}
rewriter.wrap()
}
#[test]
fn unsupported_exclude_selector_does_not_halt_indexing() {
let data = test_parse_with_exclusions(
vec!["<p>Kept</p>", "<div class='ad'>Dropped</div>"],
&[":not(.a b)", ".ad"],
);
assert_eq!(data.digest, "Kept.");
}
#[test]
fn words_weights() {
let data = test_parse(vec![