Add an opt-in option to lowercase filter values

Filter values that only differ in case, like `Mint` and `mint`, came
through as separate values. The new `--lowercase-filters` option
(`lowercase_filters`, `PAGEFIND_LOWERCASE_FILTERS`, and
`lowercaseFilters` / `lowercase_filters` in the Node and Python
wrappers) lowercases filter values when indexing and merges values on
a page that become equal. Filter names are left as they are. Off by
default.

Fixes #737

docs/content/docs/config-options.md

@@ -121,6 +121,15 @@ This defaults to false, so playground files are not written to your live site. P
 |----------------------|--------------------|--------------------|
 | `--write-playground` | `PAGEFIND_WRITE_PLAYGROUND` | `write_playground` |
 
+### Lowercase filters
+Lowercases all filter values when indexing, so that values differing only in case are merged. With this option, `<span data-pagefind-filter="garnish">Mint</span>` and `<span data-pagefind-filter="garnish">mint</span>` both come through as the `mint` value of the `garnish` filter. Filter names are left as they are.
+
+Filter values passed to Pagefind search should then be lowercase too, and any UI that displays them will show the lowercased values.
+
+| CLI Flag              | ENV Variable                 | Config Key          |
+|-----------------------|------------------------------|---------------------|
+| `--lowercase-filters` | `PAGEFIND_LOWERCASE_FILTERS` | `lowercase_filters` |
+
 ### Verbose
 Prints extra logging while indexing the site. Only affects the CLI, does not impact web-facing search.
 

docs/content/docs/node-api.md

@@ -64,6 +64,7 @@ const { index } = await pagefind.createIndex({
     excludeSelectors: [".my-code-blocks"],
     forceLanguage: "en",
     includeCharacters: "._",
+    lowercaseFilters: false,
     keepIndexUrl: false,
     writePlayground: false,
     verbose: false,

docs/content/docs/py-api.md

@@ -131,6 +131,7 @@ config = IndexConfig(
     exclude_selectors=["nav"],
     force_language="en",
     include_characters="._",
+    lowercase_filters=False,
     verbose=True,
     logfile="index.log",
     keep_index_url=True,

pagefind/integration_tests/filtering/filter-values-can-be-lowercased.toolproof.yml

@@ -0,0 +1,51 @@
+name: Filtering > Filter values can be lowercased
+steps:
+  - step: I have the environment variable "PAGEFIND_SITE" set to "public"
+  - step: I have a "public/index.html" file with the content {html}
+    html: >-
+      <!DOCTYPE html><html lang="en"><head></head><body><p
+      data-results>Nothing</p></body></html>
+  - step: I have a "public/cheeka/index.html" file with the content {html}
+    html: >-
+      <!DOCTYPE html><html lang="en"><head></head><body><span
+      data-pagefind-filter="color">Black</span>
+
+      <span data-pagefind-filter="color">White</span>
+
+      <span data-pagefind-filter="color">white</span>
+
+      <h1>Cat</h1></body></html>
+  - step: I have a "public/theodore/index.html" file with the content {html}
+    html: >-
+      <!DOCTYPE html><html lang="en"><head></head><body><span
+      data-pagefind-filter="color">WHITE</span>
+
+      <h1>Cat</h1></body></html>
+  - macro: I run Pagefind with "--lowercase-filters"
+  - step: stdout should contain "Running Pagefind"
+  - step: I serve the directory "public"
+  - step: In my browser, I load "/"
+  - step: In my browser, I evaluate {js}
+    js: |-
+      let pagefind = await import("/pagefind/pagefind.js");
+
+      let filters = await pagefind.filters();
+      let strings = Object.entries(filters).map(([filter, values]) => {
+          values = Object.entries(values).map(([value, count]) => {
+              return `${value}(${count})`;
+          })
+          return `${filter}:[${values.join(", ")}]`;
+      });
+
+      let search = await pagefind.search("Cat", { filters: { color: "white" } });
+      let data = await Promise.all(search.results.map(result => result.data()));
+
+      document.querySelector('[data-results]').innerText = strings.join(' ')
+          + ' ' + data.map(d => d.url).sort().join(', ');
+  - step: In my browser, the console should be empty
+  - step: In my browser, I evaluate {js}
+    js: >-
+      let val = await toolproof.querySelector("[data-results]");
+
+      toolproof.assert_eq(val.innerHTML, `color:[black(1), white(2)]
+      /cheeka/, /theodore/`);

pagefind/integration_tests/node_api/node_base/pagefind-lowercase-filters-config.toolproof.yml

@@ -0,0 +1,39 @@
+name: Node API Base Tests > Pagefind lowercaseFilters config
+platforms:
+  - linux
+  - mac
+
+steps:
+  - ref: ./background.toolproof.yml
+  - step: I have a "public/index.js" file with the content {js}
+    js: |2-
+       import * as pagefind from "pagefind";
+
+       const run = async () => {
+           const { index } = await pagefind.createIndex({
+               lowercaseFilters: true,
+           });
+           await index.addHTMLFile({sourcePath: "page_a/index.html", content: "<html lang='en'><body><h1 data-pagefind-filter='color:White'>Cat</h1></body></html>"});
+           await index.addCustomRecord({
+               url: "/page_b/",
+               content: "Cat",
+               language: "en",
+               filters: { color: ["WHITE", "Black"] },
+           });
+           await index.writeFiles();
+           console.log(`Successfully wrote files`);
+       }
+
+       run();
+  - macro: I run Pagefind Node as "index.js" in "public"
+  - step: stdout should contain "Successfully wrote files"
+  - step: The file "public/pagefind/pagefind.js" should not be empty
+  - step: I serve the directory "public"
+  - step: In my browser, I load "/"
+  - step: In my browser, I evaluate {js}
+    js: |-
+      let pagefind = await import("/pagefind/pagefind.js");
+      let filters = await pagefind.filters();
+
+      toolproof.assert_eq(JSON.stringify(filters), `{"color":{"black":1,"white":2}}`);
+  - step: In my browser, the console should be empty

pagefind/src/fossick/mod.rs

@@ -281,6 +281,9 @@ impl Fossicker {
 
         let (content, word_data, anchors, word_count) = self.parse_digest(options);
         self.tidy_meta_and_filters();
+        if options.lowercase_filters {
+            self.lowercase_filters();
+        }
 
         let data = self.data.unwrap();
 
@@ -556,6 +559,22 @@ impl Fossicker {
         }
     }
 
+    /// Lowercases filter values, merging values on a page that only differed in case.
+    fn lowercase_filters(&mut self) {
+        if let Some(data) = self.data.as_mut() {
+            for values in data.filters.values_mut() {
+                let mut lowercased: Vec<String> = Vec::with_capacity(values.len());
+                for value in values.drain(..) {
+                    let value = value.to_lowercase();
+                    if !lowercased.contains(&value) {
+                        lowercased.push(value);
+                    }
+                }
+                *values = lowercased;
+            }
+        }
+    }
+
     /// Parse words from metadata fields and return them with field IDs and positions.
     fn parse_meta_words(
         meta: &BTreeMap<String, String>,
@@ -1226,6 +1245,41 @@ mod tests {
         }
     }
 
+    #[tokio::test]
+    async fn lowercase_filters() {
+        let html = [
+            "<html><body>",
+            "<span data-pagefind-filter=\"Garnish\">Mint</span>",
+            "<span data-pagefind-filter=\"Garnish\">mint</span>",
+            "<span data-pagefind-filter=\"Garnish\">Basil</span>",
+            "<p>Hello World!</p>",
+            "</body></html>",
+        ]
+        .concat();
+
+        let data = test_fossick(html.clone())
+            .fossick_sync(&test_opts())
+            .unwrap();
+        assert_eq!(
+            data.fragment.data.filters,
+            BTreeMap::from_iter([(
+                "Garnish".to_string(),
+                vec!["Mint".to_string(), "mint".to_string(), "Basil".to_string()]
+            )])
+        );
+
+        let mut opts = test_opts();
+        opts.lowercase_filters = true;
+        let data = test_fossick(html).fossick_sync(&opts).unwrap();
+        assert_eq!(
+            data.fragment.data.filters,
+            BTreeMap::from_iter([(
+                "Garnish".to_string(),
+                vec!["mint".to_string(), "basil".to_string()]
+            )])
+        );
+    }
+
     #[cfg(not(target_os = "windows"))]
     #[test]
     fn building_url() {

pagefind/src/options.rs

@@ -96,6 +96,14 @@ pub(crate) struct PagefindInboundConfig {
     #[clap(required = false)]
     pub(crate) include_characters: Option<String>,
 
+    #[clap(
+        long,
+        help = "Lowercase filter values when indexing, so that values differing only in case are merged into one."
+    )]
+    #[clap(required = false)]
+    #[serde(default = "defaults::default_false")]
+    pub(crate) lowercase_filters: bool,
+
     #[clap(
         long,
         help = "Serve the source directory after creating the search index"
@@ -199,6 +207,8 @@ pub struct PagefindServiceConfig {
     #[patch(as_option)]
     /// Include these characters when indexing and searching words.
     pub(crate) include_characters: Option<String>,
+    /// Lowercase filter values when indexing, so that values differing only in case are merged into one.
+    pub(crate) lowercase_filters: Option<bool>,
 }
 
 mod defaults {
@@ -230,6 +240,7 @@ pub(crate) struct SearchOptions {
     pub(crate) glob: String,
     pub(crate) force_language: Option<String>,
     pub(crate) include_characters: Vec<char>,
+    pub(crate) lowercase_filters: bool,
     pub(crate) version: &'static str,
     pub(crate) logger: Logger,
     pub(crate) keep_index_url: bool,
@@ -311,6 +322,7 @@ impl SearchOptions {
                 glob: config.glob,
                 force_language: config.force_language,
                 include_characters,
+                lowercase_filters: config.lowercase_filters,
                 version: env!("CARGO_PKG_VERSION"),
                 logger: Logger::new(
                     log_level,

wrappers/node/lib/index.js

@@ -59,6 +59,7 @@ export const createIndex = (config) => new Promise((resolve, reject) => {
                 keep_index_url: config?.keepIndexUrl,
                 write_playground: config?.writePlayground,
                 include_characters: config?.includeCharacters,
+                lowercase_filters: config?.lowercaseFilters,
             }
         }, (response) => {
             /** @type {function(InternalResponsePayload): Omit<NewIndexResponse, 'errors'>?} */

wrappers/node/types/index.d.ts

@@ -56,6 +56,10 @@ export interface PagefindServiceConfig {
      * @example "<>$"
      */
     includeCharacters?: string,
+    /**
+     * Lowercase filter values when indexing, so that values differing only in case are merged into one.
+     */
+    lowercaseFilters?: boolean,
 }
 
 

wrappers/node/types/internal.d.ts

@@ -34,6 +34,7 @@ export interface InternalPagefindServiceConfig {
     keep_index_url?: boolean,
     write_playground?: boolean,
     include_characters?: string,
+    lowercase_filters?: boolean,
 }
 
 export interface InternalAddFileRequest {

wrappers/python/src/pagefind/index/__init__.py

@@ -59,6 +59,8 @@ class IndexConfig(TypedDict, total=False):
 
     Useful for sites documenting technical topics such as programming languages.
     """
+    lowercase_filters: Optional[bool]
+    """Lowercase filter values when indexing, so that values differing only in case are merged into one."""
     output_path: Optional[str]
     """
     The folder to output the search bundle into, relative to the processed site.

wrappers/python/src/pagefind/service/types.py

@@ -22,6 +22,7 @@ class InternalPagefindServiceConfig(TypedDict, total=False):
     keep_index_url: Optional[bool]
     write_playground: Optional[bool]
     include_characters: Optional[str]
+    lowercase_filters: Optional[bool]
 
 
 class InternalNewIndexRequest(TypedDict):