Upstream Pagefind/pagefind state of the touched files before e920ec18c5

docs/content/docs/config-options.md

@@ -63,7 +63,7 @@ exclude_selectors:
   - "[id^='prefix-']"
 ```
 
-All children will also be excluded, so using a `#my_navigation *` selector is not required. In other words, the semantics are the same as the [data-pagefind-ignore="all"](/docs/indexing/#removing-individual-elements-from-the-index) attribute: matched elements are skipped for metadata and filters as well as for content.
+All children will also be ignored, so using a `#my_navigation *` selector is not required — in other words, the semantics are the same as the [data-pagefind-ignore](/docs/indexing/#removing-individual-elements-from-the-index) attribute.
 
 Note that currently Pagefind only supports lists of options via configuration files. If using the `--exclude-selectors` CLI flag or the `PAGEFIND_EXCLUDE_SELECTORS` environment variable, only one selector may be supplied. The selector may be a comma-separated CSS selector though, so the above example can be passed as `--exclude-selectors "#my_navigation, blockquote > span, [id^='prefix-']"`.
 

pagefind/src/fossick/parser.rs

@@ -1,6 +1,6 @@
 use lazy_static::lazy_static;
 use lol_html::html_content::Element;
-use lol_html::{element, text, HtmlRewriter, Settings};
+use lol_html::{element, text, HtmlRewriter, Selector, Settings};
 use regex::Regex;
 use std::cell::RefCell;
 use std::collections::BTreeMap;
@@ -63,9 +63,6 @@ struct DomParserData {
     has_html_element: bool,
     has_old_bundle_reference: bool,
     has_default_ui_reference: bool,
-    // Set when an --exclude-selectors match fires, and taken by the handler that
-    // builds the node for that same element.
-    exclude_element: bool,
 }
 
 #[derive(Clone, Copy, Debug, PartialEq)]
@@ -137,28 +134,28 @@ impl<'a> DomParser<'a> {
     pub fn new(options: &'a SearchOptions) -> Self {
         let data = Rc::new(RefCell::new(DomParserData::default()));
         let root = format!("{}, {} *", options.root_selector, options.root_selector);
-        let scope_to_root = |selectors: &mut dyn Iterator<Item = &str>| {
+        let mut custom_exclusions = options.exclude_selectors.clone();
-            selectors
+        custom_exclusions.extend(REMOVE_SELECTORS.iter().map(|s| s.to_string()));
-                .map(|e| format!("{} {}", options.root_selector, e))
+        let custom_exclusions = custom_exclusions
+            .iter()
+            .filter_map(|e| {
+                let selector = format!("{} {}", options.root_selector, e);
+                match selector.parse::<Selector>() {
+                    Ok(_) => Some(selector),
+                    Err(error) => {
+                        options.logger.warn(format!(
+                            "Ignoring the exclude selector {e:?}, which Pagefind can't parse: {error}. Indexing will continue without it."
+                        ));
+                        None
+                    }
+                }
+            })
             .collect::<Vec<_>>()
-                .join(", ")
+            .join(", ");
-        };
-        let user_exclusions =
-            scope_to_root(&mut options.exclude_selectors.iter().map(String::as_str));
-        let auto_exclusions = scope_to_root(&mut REMOVE_SELECTORS.iter().copied());
         let mut anchor_counter = 0;
 
-        // Registered ahead of the handler below so that the element's own status
-        // is known before any of its metadata is read.
-        let user_exclusion_handler = (!user_exclusions.is_empty()).then(|| {
-            enclose! { (data) element!(user_exclusions, move |_el| {
-                data.borrow_mut().exclude_element = true;
-                Ok(())
-            })}
-        });
-
         let rewriter = HtmlRewriter::new(
-            user_exclusion_handler.into_iter().chain(vec![
+            vec![
                     enclose! { (data) element!("html", move |el| {
                         let mut data = data.borrow_mut();
                         data.has_html_element = true;
@@ -168,7 +165,6 @@ impl<'a> DomParser<'a> {
                         Ok(())
                     })},
                     enclose! { (data) element!(root, move |el| {
-                        let excluded_by_selector = std::mem::take(&mut data.borrow_mut().exclude_element);
                         let explicit_ignore_flag = el.get_attribute("data-pagefind-ignore").map(|attr| {
                             match attr.to_ascii_lowercase().as_str() {
                                 "" | "index" | "true" => NodeStatus::Ignored,
@@ -190,9 +186,7 @@ impl<'a> DomParser<'a> {
                         let index_attrs: Option<Vec<String>> = el.get_attribute("data-pagefind-index-attrs").map(|attr| attr.split(',').map(|a| a.trim().to_string()).collect());
                         let tag_name = el.tag_name();
 
-                        let status = if excluded_by_selector {
+                        let status = if treat_as_body {
-                            NodeStatus::Excluded
-                        } else if treat_as_body {
                             NodeStatus::Body
                         } else if let Some(explicit_ignore_flag) = explicit_ignore_flag {
                             explicit_ignore_flag
@@ -512,9 +506,8 @@ impl<'a> DomParser<'a> {
                         }
                         Ok(())
                     })},
-                    // Elements we always skip stay "ignored" rather than "excluded",
+                    // If we hit a selector that should be excluded, mark whatever the current node is as such
-                    // so that metadata tagged inside a <head> or a <nav> still counts.
+                    enclose! { (data) element!(custom_exclusions, move |_el| {
-                    enclose! { (data) element!(auto_exclusions, move |_el| {
                         let data = data.borrow_mut();
                         let mut node = data.current_node.borrow_mut();
                         node.status = NodeStatus::Ignored;
@@ -570,7 +563,8 @@ impl<'a> DomParser<'a> {
                         }
                         Ok(())
                     })},
-            ])
+            ]
+            .into_iter()
             .fold(Settings::new().with_strict(false), |settings, handler| {
                 settings.append_element_content_handler(handler)
             }),
@@ -728,18 +722,13 @@ mod tests {
     }
 
     fn test_raw_parse(input: Vec<&'static str>) -> DomParserResult {
-        test_raw_parse_with_args(vec![], input)
-    }
-
-    fn test_raw_parse_with_args(
-        extra_args: Vec<&'static str>,
-        input: Vec<&'static str>,
-    ) -> DomParserResult {
         use clap::CommandFactory;
-        let mut args = vec!["pagefind", "--source", "not_important"];
-        args.extend(extra_args);
         let config_args = vec![twelf::Layer::Clap(
-            crate::PagefindInboundConfig::command().get_matches_from(args),
+            crate::PagefindInboundConfig::command().get_matches_from(vec![
+                "pagefind",
+                "--source",
+                "not_important",
+            ]),
         )];
         let config =
             SearchOptions::load(crate::PagefindInboundConfig::with_layers(&config_args).unwrap())
@@ -757,6 +746,41 @@ mod tests {
         test_raw_parse(input)
     }
 
+    fn test_parse_with_exclusions(
+        mut input: Vec<&'static str>,
+        exclusions: &[&'static str],
+    ) -> DomParserResult {
+        use clap::CommandFactory;
+        let mut args = vec!["pagefind", "--source", "not_important"];
+        for exclusion in exclusions {
+            args.push("--exclude-selectors");
+            args.push(exclusion);
+        }
+        let config_args = vec![twelf::Layer::Clap(
+            crate::PagefindInboundConfig::command().get_matches_from(args),
+        )];
+        let config =
+            SearchOptions::load(crate::PagefindInboundConfig::with_layers(&config_args).unwrap())
+                .unwrap();
+        let mut rewriter = DomParser::new(&config);
+        input.insert(0, "<html><body>");
+        input.push("</body></html>");
+        for line in input {
+            let _ = rewriter.write(line.as_bytes());
+        }
+        rewriter.wrap()
+    }
+
+    #[test]
+    fn unsupported_exclude_selector_does_not_halt_indexing() {
+        let data = test_parse_with_exclusions(
+            vec!["<p>Kept</p>", "<div class='ad'>Dropped</div>"],
+            &[":not(.a b)", ".ad"],
+        );
+
+        assert_eq!(data.digest, "Kept.");
+    }
+
     #[test]
     fn words_weights() {
         let data = test_parse(vec![
@@ -842,45 +866,6 @@ mod tests {
         assert_eq!(data.digest, "Elements like: forms. As well as *crickets*.");
     }
 
-    #[test]
-    fn excluded_selectors_are_excluded_entirely() {
-        let data = test_raw_parse_with_args(
-            vec!["--exclude-selectors", "img.emoji, .promo"],
-            vec![
-                "<html><body>",
-                "<h1>Kitty blog</h1>",
-                "<img class='emoji' alt='thumbs up' src='/emoji/+1.png'>",
-                "<div class='promo' data-pagefind-meta='deal:half price'></div>",
-                "<div class='promo' data-pagefind-filter='color:red'>Buy now</div>",
-                "<img alt='a kitty' src='/kitty.jpg'>",
-                "</body></html>",
-            ],
-        );
-
-        assert_eq!(
-            data.digest,
-            "___PAGEFIND_AUTO_WEIGHT___7 Kitty blog. ___END_PAGEFIND_WEIGHT___"
-        );
-        assert_eq!(data.meta.get("deal"), None);
-        assert_eq!(data.filters.get("color"), None);
-        assert_eq!(data.meta.get("image"), Some(&"/kitty.jpg".to_owned()));
-        assert_eq!(data.meta.get("image_alt"), Some(&"a kitty".to_owned()));
-    }
-
-    #[test]
-    fn automatically_removed_elements_still_supply_metadata() {
-        let data = test_raw_parse(vec![
-            "<html><head><title>Kitty blog</title></head><body>",
-            "<nav data-pagefind-meta='section:navigation'>Home</nav>",
-            "<p>Meow</p>",
-            "</body></html>",
-        ]);
-
-        assert_eq!(data.digest, "Meow.");
-        assert_eq!(data.meta.get("section"), Some(&"navigation".to_owned()));
-        assert_eq!(data.meta.get("title"), Some(&"Kitty blog".to_owned()));
-    }
-
     #[test]
     fn return_metadata() {
         let data = test_raw_parse(vec![

pagefind/src/options.rs

@@ -0,0 +1,368 @@
+//! Configuration that can be supplied to the `api` module when using Pagefind as a service.
+
+use anyhow::{bail, Result};
+use clap::Parser;
+use rust_patch::Patch;
+use serde::{Deserialize, Serialize};
+use std::{env, path::PathBuf};
+use twelf::config;
+use typed_builder::TypedBuilder;
+
+use crate::{
+    logging::{LogLevel, Logger},
+    utils::WORD_SYMBOLS,
+};
+
+//
+// If editing this configuration struct,
+// also make sure to edit the patch config below,
+// (if it makes sense for that option to be set via the service),
+// and also update any wrapper packages to include definitions for that option.
+//
+// No options should be added that are required.
+//
+
+#[config]
+#[derive(Parser, Debug, Clone)]
+#[clap(author, version, about, long_about = None)]
+pub(crate) struct PagefindInboundConfig {
+    #[clap(long, help = "DEPRECATED: Use the `site` option instead")]
+    #[clap(required = false, hide = true)]
+    #[serde(default)] // This is actually required, but we validate that later
+    pub(crate) source: String,
+
+    #[clap(long, short, help = "The location of your built static website")]
+    #[clap(required = false)]
+    #[serde(default)] // This is actually required, but we validate that later
+    pub(crate) site: String,
+
+    #[clap(
+        long,
+        short,
+        help = "DEPRECATED: Use `output_subdir` or `output_path` instead"
+    )]
+    #[clap(required = false, hide = true)]
+    pub(crate) bundle_dir: Option<String>,
+
+    #[clap(
+        long,
+        help = "Where to output the search bundle, relative to the processed site"
+    )]
+    #[clap(required = false)]
+    pub(crate) output_subdir: Option<String>,
+
+    #[clap(
+        long,
+        help = "Where to output the search bundle, relative to the working directory of the command"
+    )]
+    #[clap(required = false)]
+    pub(crate) output_path: Option<String>,
+
+    #[clap(
+        long,
+        help = "The element Pagefind should treat as the root of the document. Usually you will want to use the data-pagefind-body attribute instead."
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_root_selector")]
+    pub(crate) root_selector: String,
+
+    #[clap(
+        long,
+        help = "Custom selectors that Pagefind should ignore when indexing. Usually you will want to use the data-pagefind-ignore attribute instead."
+    )]
+    #[clap(required = false)]
+    #[serde(default)]
+    pub(crate) exclude_selectors: Vec<String>,
+
+    #[clap(
+        long,
+        help = "The file glob Pagefind uses to find HTML files. Defaults to \"**/*.{html}\""
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_glob")]
+    pub(crate) glob: String,
+
+    #[clap(
+        long,
+        help = "Ignore any detected languages and index the whole site as a single language. Expects an ISO 639-1 code."
+    )]
+    #[clap(required = false)]
+    pub(crate) force_language: Option<String>,
+
+    #[clap(
+        long,
+        help = "Include these characters when indexing and searching words. Useful for sites documenting technical topics such as programming languages."
+    )]
+    #[clap(required = false)]
+    pub(crate) include_characters: Option<String>,
+
+    #[clap(
+        long,
+        help = "Serve the source directory after creating the search index"
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_false")]
+    pub(crate) serve: bool,
+
+    #[clap(
+        long,
+        short,
+        help = "Print verbose logging while indexing the site. Does not impact the web-facing search."
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_false")]
+    pub(crate) verbose: bool,
+
+    #[clap(
+        long,
+        short,
+        help = "Only log errors and warnings while indexing the site. Does not impact the web-facing search."
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_false")]
+    pub(crate) quiet: bool,
+
+    #[clap(
+        long,
+        help = "Only log errors while indexing the site. Does not impact the web-facing search."
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_false")]
+    pub(crate) silent: bool,
+
+    #[clap(
+        long,
+        short,
+        help = "Path to a logfile to write to. Will replace the file on each run"
+    )]
+    #[clap(required = false)]
+    #[serde(default)]
+    pub(crate) logfile: Option<String>,
+
+    #[clap(
+        long,
+        short,
+        help = "Keep \"index.html\" at the end of search result paths. Defaults to false, stripping \"index.html\"."
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_false")]
+    pub(crate) keep_index_url: bool,
+
+    #[clap(
+        long,
+        help = "Output the Pagefind Playground to <bundle_dir>/playground/ when building the search index. By default, this is only available via --serve."
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_false")]
+    pub(crate) write_playground: bool,
+
+    #[clap(long)]
+    #[clap(required = false, hide = true)]
+    #[serde(default = "defaults::default_false")]
+    pub(crate) service: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, Patch, TypedBuilder)]
+#[patch = "PagefindInboundConfig"]
+#[builder(
+    doc,
+    field_defaults(default, setter(strip_option)),
+    builder_method(
+        vis = "pub",
+        doc = "Create a builder for building `PagefindServiceConfig` for the api."
+    ),
+    builder_type(vis = "pub"),
+    build_method(vis = "pub")
+)]
+/// Fields that can be set via the Pagefind service.
+/// In other words, the subset of the Pagefind configuration that makes sense to set globally,
+/// excluding fields that are irrelevant or set when each individual method is called.
+///
+/// Must be constructed through the `PagefindServiceConfigBuilder` interface.
+pub struct PagefindServiceConfig {
+    /// The element Pagefind should treat as the root of the document.
+    pub(crate) root_selector: Option<String>,
+    /// Custom selectors that Pagefind should ignore when indexing.
+    pub(crate) exclude_selectors: Option<Vec<String>>,
+    #[patch(as_option)]
+    /// Ignore any detected languages and index the whole site as a single language. Expects an ISO 639-1 code.
+    pub(crate) force_language: Option<String>,
+    /// Print verbose logging while indexing the site. Does not impact the web-facing search.
+    pub(crate) verbose: Option<bool>,
+    #[patch(as_option)]
+    /// Path to a logfile to write to. Will replace the file on each run
+    pub(crate) logfile: Option<String>,
+    /// Keep \"index.html\" at the end of search result paths. Defaults to false, stripping \"index.html\".
+    pub(crate) keep_index_url: Option<bool>,
+    /// Output the Pagefind Playground to <bundle_dir>/playground/ when building the search index.
+    pub(crate) write_playground: Option<bool>,
+    #[patch(as_option)]
+    /// Include these characters when indexing and searching words.
+    pub(crate) include_characters: Option<String>,
+}
+
+mod defaults {
+    pub fn default_bundle_dir() -> String {
+        "pagefind".into()
+    }
+    pub fn default_root_selector() -> String {
+        "html".into()
+    }
+    pub fn default_glob() -> String {
+        "**/*.{html}".into()
+    }
+    pub fn default_false() -> bool {
+        false
+    }
+    pub fn default_index_chunk_size() -> usize {
+        20000
+    }
+}
+
+// The configuration object used internally
+#[derive(Debug, Clone)]
+pub(crate) struct SearchOptions {
+    pub(crate) working_directory: PathBuf,
+    pub(crate) site_source: PathBuf,
+    pub(crate) bundle_output: PathBuf,
+    pub(crate) root_selector: String,
+    pub(crate) exclude_selectors: Vec<String>,
+    pub(crate) glob: String,
+    pub(crate) force_language: Option<String>,
+    pub(crate) include_characters: Vec<char>,
+    pub(crate) version: &'static str,
+    pub(crate) logger: Logger,
+    pub(crate) keep_index_url: bool,
+    pub(crate) running_as_service: bool,
+    pub(crate) write_playground: bool,
+    pub(crate) config_warnings: ConfigWarnings,
+    pub(crate) index_chunk_size: usize,
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct ConfigWarnings {
+    pub(crate) unconfigured_bundle_output: bool,
+    pub(crate) using_deprecated_source: bool,
+    pub(crate) using_deprecated_bundle_dir: bool,
+}
+
+impl SearchOptions {
+    pub(crate) fn load(config: PagefindInboundConfig) -> Result<Self> {
+        if !config.service && config.site.is_empty() && config.source.is_empty() {
+            eprintln!("Required argument site not supplied. Pagefind needs to know the root of your built static site.");
+            eprintln!("Provide a --site flag, a PAGEFIND_SITE environment variable, or a site key in a Pagefind configuration file.");
+            bail!("Missing argument: site");
+        } else {
+            let log_level = if config.verbose {
+                LogLevel::Verbose
+            } else if config.quiet {
+                LogLevel::Quiet
+            } else if config.silent {
+                LogLevel::Silent
+            } else {
+                LogLevel::Standard
+            };
+
+            let log_to_terminal = !config.service;
+            let working_directory = env::current_dir().unwrap();
+
+            let site_source = if !config.site.is_empty() {
+                working_directory.join(PathBuf::from(config.site))
+            } else {
+                working_directory.join(PathBuf::from(config.source.clone()))
+            };
+
+            // For backwards compat pre-1.0, we output files for older defaults
+            // when the path hasn't been set
+            let configured_bundle_output = config
+                .output_path
+                .as_ref()
+                .or(config.output_subdir.as_ref())
+                .or(config.bundle_dir.as_ref())
+                .is_some();
+
+            let warnings = ConfigWarnings {
+                unconfigured_bundle_output: !configured_bundle_output,
+                using_deprecated_source: !config.source.is_empty(),
+                using_deprecated_bundle_dir: config.bundle_dir.is_some(),
+            };
+
+            let bundle_output = if let Some(subdir) = config.output_path {
+                working_directory.join(subdir)
+            } else {
+                let subdir = config
+                    .output_subdir
+                    .or(config.bundle_dir)
+                    .unwrap_or_else(|| defaults::default_bundle_dir());
+                site_source.join(subdir)
+            };
+
+            let mut include_characters = WORD_SYMBOLS.to_vec();
+            if let Some(custom_include_characters) = config.include_characters {
+                include_characters.extend(custom_include_characters.chars());
+            }
+
+            Ok(Self {
+                working_directory,
+                site_source,
+                bundle_output,
+                root_selector: config.root_selector,
+                exclude_selectors: config.exclude_selectors,
+                glob: config.glob,
+                force_language: config.force_language,
+                include_characters,
+                version: env!("CARGO_PKG_VERSION"),
+                logger: Logger::new(
+                    log_level,
+                    log_to_terminal,
+                    config.logfile.map(PathBuf::from),
+                ),
+                keep_index_url: config.keep_index_url,
+                running_as_service: config.service,
+                write_playground: config.write_playground,
+                config_warnings: warnings,
+                index_chunk_size: env::var("PAGEFIND_UNSTABLE_INDEX_CHUNK_SIZE")
+                    .ok()
+                    .and_then(|s| s.parse().ok())
+                    .unwrap_or(defaults::default_index_chunk_size()),
+            })
+        }
+    }
+}
+
+impl ConfigWarnings {
+    pub(crate) fn get_strings(&self) -> Vec<String> {
+        let mut strings = vec![];
+        if self.using_deprecated_bundle_dir {
+            strings.push(
+                "\n\
+                 The `bundle-dir` option is deprecated as of Pagefind 1.0. \
+                 Use either `output-subdir` or `output-path` instead:\n\n\
+                 cli:    --output-subdir\n\
+                 config: output_subdir\n\
+                 env:    PAGEFIND_OUTPUT_SUBDIR\n\
+                 └─ \"Where to output the search files, relative to the processed site\"\n\n\
+                 cli:    --output-path\n\
+                 config: output_path\n\
+                 env:    PAGEFIND_OUTPUT_PATH\n\
+                 └─ \"Where to output the search files, relative to the working directory of the command\"\n"
+                    .into(),
+            );
+        }
+
+        if self.using_deprecated_source {
+            strings.push(
+                "\n\
+                 The `source` option is deprecated as of Pagefind 1.0. \
+                 The `source` option has been renamed to `site`:\n\n\
+                 cli:    --site\n\
+                 config: site\n\
+                 env:    PAGEFIND_SITE\n\
+                 └─ \"The location of your built static website\"\n"
+                    .into(),
+            );
+        }
+
+        strings
+    }
+}