From 3a1d31147a9f53afe63d5e372e6c09ec6806cca2 Mon Sep 17 00:00:00 2001 From: DualFroz Date: Sat, 3 Oct 2026 01:43:17 +0200 Subject: [PATCH] Add an opt-in option to lowercase filter values Filter values that only differ in case, like `Mint` and `mint`, came through as separate values. The new `--lowercase-filters` option (`lowercase_filters`, `PAGEFIND_LOWERCASE_FILTERS`, and `lowercaseFilters` / `lowercase_filters` in the Node and Python wrappers) lowercases filter values when indexing and merges values on a page that become equal. Filter names are left as they are. Off by default. Fixes #737 --- docs/content/docs/config-options.md | 9 ++++ docs/content/docs/node-api.md | 1 + docs/content/docs/py-api.md | 1 + ...ter-values-can-be-lowercased.toolproof.yml | 51 ++++++++++++++++++ ...ind-lowercase-filters-config.toolproof.yml | 39 ++++++++++++++ pagefind/src/fossick/mod.rs | 54 +++++++++++++++++++ pagefind/src/options.rs | 12 +++++ wrappers/node/lib/index.js | 1 + wrappers/node/types/index.d.ts | 4 ++ wrappers/node/types/internal.d.ts | 1 + .../python/src/pagefind/index/__init__.py | 2 + wrappers/python/src/pagefind/service/types.py | 1 + 12 files changed, 176 insertions(+) create mode 100644 pagefind/integration_tests/filtering/filter-values-can-be-lowercased.toolproof.yml create mode 100644 pagefind/integration_tests/node_api/node_base/pagefind-lowercase-filters-config.toolproof.yml diff --git a/docs/content/docs/config-options.md b/docs/content/docs/config-options.md index b3eedce..d633a4d 100644 --- a/docs/content/docs/config-options.md +++ b/docs/content/docs/config-options.md @@ -121,6 +121,15 @@ This defaults to false, so playground files are not written to your live site. P |----------------------|--------------------|--------------------| | `--write-playground` | `PAGEFIND_WRITE_PLAYGROUND` | `write_playground` | +### Lowercase filters +Lowercases all filter values when indexing, so that values differing only in case are merged. With this option, `Mint` and `mint` both come through as the `mint` value of the `garnish` filter. Filter names are left as they are. + +Filter values passed to Pagefind search should then be lowercase too, and any UI that displays them will show the lowercased values. + +| CLI Flag | ENV Variable | Config Key | +|-----------------------|------------------------------|---------------------| +| `--lowercase-filters` | `PAGEFIND_LOWERCASE_FILTERS` | `lowercase_filters` | + ### Verbose Prints extra logging while indexing the site. Only affects the CLI, does not impact web-facing search. diff --git a/docs/content/docs/node-api.md b/docs/content/docs/node-api.md index 1da3dd3..e7817a1 100644 --- a/docs/content/docs/node-api.md +++ b/docs/content/docs/node-api.md @@ -64,6 +64,7 @@ const { index } = await pagefind.createIndex({ excludeSelectors: [".my-code-blocks"], forceLanguage: "en", includeCharacters: "._", + lowercaseFilters: false, keepIndexUrl: false, writePlayground: false, verbose: false, diff --git a/docs/content/docs/py-api.md b/docs/content/docs/py-api.md index 55954b5..d5d0fd9 100644 --- a/docs/content/docs/py-api.md +++ b/docs/content/docs/py-api.md @@ -131,6 +131,7 @@ config = IndexConfig( exclude_selectors=["nav"], force_language="en", include_characters="._", + lowercase_filters=False, verbose=True, logfile="index.log", keep_index_url=True, diff --git a/pagefind/integration_tests/filtering/filter-values-can-be-lowercased.toolproof.yml b/pagefind/integration_tests/filtering/filter-values-can-be-lowercased.toolproof.yml new file mode 100644 index 0000000..285f59a --- /dev/null +++ b/pagefind/integration_tests/filtering/filter-values-can-be-lowercased.toolproof.yml @@ -0,0 +1,51 @@ +name: Filtering > Filter values can be lowercased +steps: + - step: I have the environment variable "PAGEFIND_SITE" set to "public" + - step: I have a "public/index.html" file with the content {html} + html: >- +

Nothing

+ - step: I have a "public/cheeka/index.html" file with the content {html} + html: >- + Black + + White + + white + +

Cat

+ - step: I have a "public/theodore/index.html" file with the content {html} + html: >- + WHITE + +

Cat

+ - macro: I run Pagefind with "--lowercase-filters" + - step: stdout should contain "Running Pagefind" + - step: I serve the directory "public" + - step: In my browser, I load "/" + - step: In my browser, I evaluate {js} + js: |- + let pagefind = await import("/pagefind/pagefind.js"); + + let filters = await pagefind.filters(); + let strings = Object.entries(filters).map(([filter, values]) => { + values = Object.entries(values).map(([value, count]) => { + return `${value}(${count})`; + }) + return `${filter}:[${values.join(", ")}]`; + }); + + let search = await pagefind.search("Cat", { filters: { color: "white" } }); + let data = await Promise.all(search.results.map(result => result.data())); + + document.querySelector('[data-results]').innerText = strings.join(' ') + + ' ' + data.map(d => d.url).sort().join(', '); + - step: In my browser, the console should be empty + - step: In my browser, I evaluate {js} + js: >- + let val = await toolproof.querySelector("[data-results]"); + + toolproof.assert_eq(val.innerHTML, `color:[black(1), white(2)] + /cheeka/, /theodore/`); diff --git a/pagefind/integration_tests/node_api/node_base/pagefind-lowercase-filters-config.toolproof.yml b/pagefind/integration_tests/node_api/node_base/pagefind-lowercase-filters-config.toolproof.yml new file mode 100644 index 0000000..5e6aa85 --- /dev/null +++ b/pagefind/integration_tests/node_api/node_base/pagefind-lowercase-filters-config.toolproof.yml @@ -0,0 +1,39 @@ +name: Node API Base Tests > Pagefind lowercaseFilters config +platforms: + - linux + - mac + +steps: + - ref: ./background.toolproof.yml + - step: I have a "public/index.js" file with the content {js} + js: |2- + import * as pagefind from "pagefind"; + + const run = async () => { + const { index } = await pagefind.createIndex({ + lowercaseFilters: true, + }); + await index.addHTMLFile({sourcePath: "page_a/index.html", content: "

Cat

"}); + await index.addCustomRecord({ + url: "/page_b/", + content: "Cat", + language: "en", + filters: { color: ["WHITE", "Black"] }, + }); + await index.writeFiles(); + console.log(`Successfully wrote files`); + } + + run(); + - macro: I run Pagefind Node as "index.js" in "public" + - step: stdout should contain "Successfully wrote files" + - step: The file "public/pagefind/pagefind.js" should not be empty + - step: I serve the directory "public" + - step: In my browser, I load "/" + - step: In my browser, I evaluate {js} + js: |- + let pagefind = await import("/pagefind/pagefind.js"); + let filters = await pagefind.filters(); + + toolproof.assert_eq(JSON.stringify(filters), `{"color":{"black":1,"white":2}}`); + - step: In my browser, the console should be empty diff --git a/pagefind/src/fossick/mod.rs b/pagefind/src/fossick/mod.rs index 9373047..3bbe34a 100644 --- a/pagefind/src/fossick/mod.rs +++ b/pagefind/src/fossick/mod.rs @@ -281,6 +281,9 @@ impl Fossicker { let (content, word_data, anchors, word_count) = self.parse_digest(options); self.tidy_meta_and_filters(); + if options.lowercase_filters { + self.lowercase_filters(); + } let data = self.data.unwrap(); @@ -556,6 +559,22 @@ impl Fossicker { } } + /// Lowercases filter values, merging values on a page that only differed in case. + fn lowercase_filters(&mut self) { + if let Some(data) = self.data.as_mut() { + for values in data.filters.values_mut() { + let mut lowercased: Vec = Vec::with_capacity(values.len()); + for value in values.drain(..) { + let value = value.to_lowercase(); + if !lowercased.contains(&value) { + lowercased.push(value); + } + } + *values = lowercased; + } + } + } + /// Parse words from metadata fields and return them with field IDs and positions. fn parse_meta_words( meta: &BTreeMap, @@ -1226,6 +1245,41 @@ mod tests { } } + #[tokio::test] + async fn lowercase_filters() { + let html = [ + "", + "Mint", + "mint", + "Basil", + "

Hello World!

", + "", + ] + .concat(); + + let data = test_fossick(html.clone()) + .fossick_sync(&test_opts()) + .unwrap(); + assert_eq!( + data.fragment.data.filters, + BTreeMap::from_iter([( + "Garnish".to_string(), + vec!["Mint".to_string(), "mint".to_string(), "Basil".to_string()] + )]) + ); + + let mut opts = test_opts(); + opts.lowercase_filters = true; + let data = test_fossick(html).fossick_sync(&opts).unwrap(); + assert_eq!( + data.fragment.data.filters, + BTreeMap::from_iter([( + "Garnish".to_string(), + vec!["mint".to_string(), "basil".to_string()] + )]) + ); + } + #[cfg(not(target_os = "windows"))] #[test] fn building_url() { diff --git a/pagefind/src/options.rs b/pagefind/src/options.rs index 2de8d3a..3c09730 100644 --- a/pagefind/src/options.rs +++ b/pagefind/src/options.rs @@ -96,6 +96,14 @@ pub(crate) struct PagefindInboundConfig { #[clap(required = false)] pub(crate) include_characters: Option, + #[clap( + long, + help = "Lowercase filter values when indexing, so that values differing only in case are merged into one." + )] + #[clap(required = false)] + #[serde(default = "defaults::default_false")] + pub(crate) lowercase_filters: bool, + #[clap( long, help = "Serve the source directory after creating the search index" @@ -199,6 +207,8 @@ pub struct PagefindServiceConfig { #[patch(as_option)] /// Include these characters when indexing and searching words. pub(crate) include_characters: Option, + /// Lowercase filter values when indexing, so that values differing only in case are merged into one. + pub(crate) lowercase_filters: Option, } mod defaults { @@ -230,6 +240,7 @@ pub(crate) struct SearchOptions { pub(crate) glob: String, pub(crate) force_language: Option, pub(crate) include_characters: Vec, + pub(crate) lowercase_filters: bool, pub(crate) version: &'static str, pub(crate) logger: Logger, pub(crate) keep_index_url: bool, @@ -311,6 +322,7 @@ impl SearchOptions { glob: config.glob, force_language: config.force_language, include_characters, + lowercase_filters: config.lowercase_filters, version: env!("CARGO_PKG_VERSION"), logger: Logger::new( log_level, diff --git a/wrappers/node/lib/index.js b/wrappers/node/lib/index.js index df28eb1..71300d2 100644 --- a/wrappers/node/lib/index.js +++ b/wrappers/node/lib/index.js @@ -59,6 +59,7 @@ export const createIndex = (config) => new Promise((resolve, reject) => { keep_index_url: config?.keepIndexUrl, write_playground: config?.writePlayground, include_characters: config?.includeCharacters, + lowercase_filters: config?.lowercaseFilters, } }, (response) => { /** @type {function(InternalResponsePayload): Omit?} */ diff --git a/wrappers/node/types/index.d.ts b/wrappers/node/types/index.d.ts index 74dbaff..6ef21c8 100644 --- a/wrappers/node/types/index.d.ts +++ b/wrappers/node/types/index.d.ts @@ -56,6 +56,10 @@ export interface PagefindServiceConfig { * @example "<>$" */ includeCharacters?: string, + /** + * Lowercase filter values when indexing, so that values differing only in case are merged into one. + */ + lowercaseFilters?: boolean, } diff --git a/wrappers/node/types/internal.d.ts b/wrappers/node/types/internal.d.ts index f2cccb3..7cf5202 100644 --- a/wrappers/node/types/internal.d.ts +++ b/wrappers/node/types/internal.d.ts @@ -34,6 +34,7 @@ export interface InternalPagefindServiceConfig { keep_index_url?: boolean, write_playground?: boolean, include_characters?: string, + lowercase_filters?: boolean, } export interface InternalAddFileRequest { diff --git a/wrappers/python/src/pagefind/index/__init__.py b/wrappers/python/src/pagefind/index/__init__.py index 6180cdf..14e6ae6 100644 --- a/wrappers/python/src/pagefind/index/__init__.py +++ b/wrappers/python/src/pagefind/index/__init__.py @@ -59,6 +59,8 @@ class IndexConfig(TypedDict, total=False): Useful for sites documenting technical topics such as programming languages. """ + lowercase_filters: Optional[bool] + """Lowercase filter values when indexing, so that values differing only in case are merged into one.""" output_path: Optional[str] """ The folder to output the search bundle into, relative to the processed site. diff --git a/wrappers/python/src/pagefind/service/types.py b/wrappers/python/src/pagefind/service/types.py index fc807c0..8d99ea3 100644 --- a/wrappers/python/src/pagefind/service/types.py +++ b/wrappers/python/src/pagefind/service/types.py @@ -22,6 +22,7 @@ class InternalPagefindServiceConfig(TypedDict, total=False): keep_index_url: Optional[bool] write_playground: Optional[bool] include_characters: Optional[str] + lowercase_filters: Optional[bool] class InternalNewIndexRequest(TypedDict):