use anyhow::{bail, Result};
#[cfg(feature = "extended")]
use charabia::Segment;
use either::Either;
use flate2::read::GzDecoder;
use hashbrown::HashMap;
use lazy_static::lazy_static;
use pagefind_stem::{Algorithm, Stemmer};
use path_slash::PathExt as _;
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
use regex::Regex;
use std::collections::BTreeMap;
use std::io::BufRead;
use std::io::Error;
use std::io::Read;
use std::ops::Mul;
use std::path::{Path, PathBuf};

use crate::fossick::splitting::get_indexable_words;
use crate::fragments::{PageAnchorData, PageFragment, PageFragmentData};
use crate::SearchOptions;
use parser::DomParser;

use self::parser::DomParserResult;

lazy_static! {
    static ref NEWLINES: Regex = Regex::new("(\n|\r\n)+").unwrap();
    static ref TRIM_NEWLINES: Regex = Regex::new("^[\n\r\\s]+|[\n\r\\s]+$").unwrap();
    static ref EXTRANEOUS_SPACES: Regex = Regex::new("\\s{2,}").unwrap();
    static ref PRIVATE_PAGEFIND: Regex = Regex::new("___PAGEFIND_[\\S]+\\s?").unwrap();
}

pub mod parser;
mod splitting;

#[derive(Debug, Clone, PartialEq)]
pub struct FossickedWord {
    pub position: u32,
    pub weight: u8,
    /// The original word before diacritic normalization, if it differs from the normalized form.
    pub original_word: Option<String>,
}

#[derive(Debug, Clone, PartialEq)]
pub struct MetaFossickedWord {
    pub field_id: u16,
    pub position: u32,
    /// The original word before diacritic normalization, if it differs from the normalized form.
    pub original_word: Option<String>,
}

#[derive(Debug, Clone)]
pub struct FossickedData {
    pub url: String,
    pub fragment: PageFragment,
    pub word_data: HashMap<String, Vec<FossickedWord>>,
    pub meta_word_data: HashMap<String, Vec<MetaFossickedWord>>,
    pub sort: BTreeMap<String, String>,
    pub has_custom_body: bool,
    pub force_inclusion: bool,
    pub has_html_element: bool,
    pub has_old_bundle_reference: bool,
    pub has_default_ui_reference: bool,
    pub language: String,
}

#[derive(Debug)]
pub struct Fossicker {
    file_path: Option<PathBuf>,
    /// Built URLs should be relative to this directory
    root_path: Option<PathBuf>,
    page_url: Option<String>,
    synthetic_content: Option<String>,
    data: Option<DomParserResult>,
}

impl Fossicker {
    pub fn new_relative_to(file_path: PathBuf, root_path: PathBuf) -> Self {
        Self {
            file_path: Some(file_path),
            root_path: Some(root_path),
            page_url: None,
            synthetic_content: None,
            data: None,
        }
    }

    pub fn new_synthetic(
        file_path: Option<PathBuf>,
        page_url: Option<String>,
        contents: String,
    ) -> Self {
        Self {
            file_path,
            root_path: None,
            page_url,
            synthetic_content: Some(contents),
            data: None,
        }
    }

    pub fn new_with_data(url: String, data: DomParserResult) -> Self {
        Self {
            file_path: None,
            root_path: None,
            page_url: Some(url),
            synthetic_content: None,
            data: Some(data),
        }
    }

    fn read_file_sync(&mut self, options: &SearchOptions) -> Result<(), Error> {
        let Some(file_path) = &self.file_path else {
            return Ok(());
        };
        let file = std::fs::File::open(file_path)?;

        let mut rewriter = DomParser::new(options);

        let mut br = std::io::BufReader::new(file);
        let mut buf = [0; 20000];

        // Check for gzip magic bytes
        let is_gzip = {
            let peek = br.fill_buf()?;
            peek.len() >= 3 && peek[0] == 0x1F && peek[1] == 0x8B && peek[2] == 0x08
        };

        if is_gzip {
            let mut decoder = GzDecoder::new(br);
            loop {
                match decoder.read(&mut buf) {
                    Ok(0) => break,
                    Ok(read) => {
                        if let Err(error) = rewriter.write(&buf[..read]) {
                            options.logger.error(format!(
                                "Failed to parse file {} — skipping this file. Error:\n{error}",
                                file_path.to_str().unwrap_or("[unknown file]"),
                            ));
                            return Ok(());
                        }
                    }
                    Err(e) => {
                        options.logger.error(format!(
                            "IO error reading gzip file {}: {e}",
                            file_path.to_str().unwrap_or("[unknown file]")
                        ));
                        return Err(e);
                    }
                }
            }
        } else {
            loop {
                match br.read(&mut buf) {
                    Ok(0) => break,
                    Ok(read) => {
                        if let Err(error) = rewriter.write(&buf[..read]) {
                            options.logger.error(format!(
                                "Failed to parse file {} — skipping this file. Error:\n{error}",
                                file_path.to_str().unwrap_or("[unknown file]")
                            ));
                            return Ok(());
                        }
                    }
                    Err(e) => {
                        options.logger.error(format!(
                            "IO error reading file {}: {e}",
                            file_path.to_str().unwrap_or("[unknown file]")
                        ));
                        return Err(e);
                    }
                }
            }
        }

        let mut data = rewriter.wrap();
        if let Some(forced_language) = &options.force_language {
            data.language = forced_language.clone();
        }

        self.data = Some(data);

        Ok(())
    }

    fn read_synthetic_sync(&mut self, options: &SearchOptions) -> Result<(), Error> {
        let Some(contents) = self.synthetic_content.as_ref() else {
            return Ok(());
        };

        let mut rewriter = DomParser::new(options);

        let mut br = std::io::Cursor::new(contents.as_bytes());
        let mut buf = [0; 20000];

        loop {
            match Read::read(&mut br, &mut buf) {
                Ok(0) => break,
                Ok(read) => {
                    if let Err(error) = rewriter.write(&buf[..read]) {
                        let path_desc = self
                            .file_path
                            .as_ref()
                            .and_then(|p| p.to_str())
                            .or(self.page_url.as_deref())
                            .unwrap_or("[unknown file]");
                        options.logger.error(format!(
                            "Failed to parse file {path_desc} — skipping this file. Error:\n{error}"
                        ));
                        return Ok(());
                    }
                }
                Err(e) => {
                    let path_desc = self
                        .file_path
                        .as_ref()
                        .and_then(|p| p.to_str())
                        .or(self.page_url.as_deref())
                        .unwrap_or("[unknown file]");
                    options.logger.error(format!(
                        "IO error reading synthetic content for {path_desc}: {e}"
                    ));
                    return Err(e);
                }
            }
        }

        let mut data = rewriter.wrap();
        if let Some(forced_language) = &options.force_language {
            data.language = forced_language.clone();
        }

        self.data = Some(data);

        Ok(())
    }

    /// Retries up to MAX_RETRIES times with exponential backoff on transient IO errors.
    fn fossick_html_sync(&mut self, options: &SearchOptions) -> Result<(), std::io::Error> {
        const MAX_RETRIES: u32 = 10;

        let mut last_error = None;
        for attempt in 0..MAX_RETRIES {
            let result = if self.synthetic_content.is_some() {
                self.read_synthetic_sync(options)
            } else {
                self.read_file_sync(options)
            };

            match result {
                Ok(()) => return Ok(()),
                Err(e) => {
                    last_error = Some(e);
                    if attempt < MAX_RETRIES - 1 {
                        // Exponential backoff: 1ms, 2ms, ... 512ms
                        std::thread::sleep(std::time::Duration::from_millis(1 << attempt));
                    }
                }
            }
        }

        Err(last_error.unwrap_or_else(|| {
            std::io::Error::new(std::io::ErrorKind::Other, "Max retries exceeded")
        }))
    }

    pub fn fossick_sync(mut self, options: &SearchOptions) -> Result<FossickedData> {
        if (self.file_path.is_some() || self.synthetic_content.is_some()) && self.data.is_none() {
            if let Err(e) = self.fossick_html_sync(options) {
                let path_desc = self
                    .file_path
                    .as_ref()
                    .and_then(|p| p.to_str())
                    .or(self.page_url.as_deref())
                    .unwrap_or("[unknown file]");
                options
                    .logger
                    .error(format!("Failed to read {path_desc} after retries: {e}"));
                bail!("Failed to read {path_desc}: {e}");
            }
        }

        let (content, word_data, anchors, word_count) = self.parse_digest(options);
        self.tidy_meta_and_filters();

        let data = self.data.unwrap();

        // Get sorted list of meta field names for consistent field IDs
        let meta_field_order: Vec<String> = data.meta.keys().cloned().collect();
        let meta_word_data =
            Self::parse_meta_words(&data.meta, &meta_field_order, &data.language, options);

        // Build URL using Option combinators for cleaner logic
        let url = self
            .page_url
            .clone()
            .or_else(|| {
                self.file_path
                    .as_ref()
                    .map(|path| build_url(path, self.root_path.as_deref(), options))
            })
            .ok_or_else(|| {
                options
                    .logger
                    .error("Tried to index file with no specified URL or file path, ignoring.");
                anyhow::anyhow!("Tried to index file with no specified URL or file path, ignoring.")
            })?;

        Ok(FossickedData {
            url: url.clone(), // Clone needed since url is used in both struct and fragment.data
            has_custom_body: data.has_custom_body,
            force_inclusion: data.force_inclusion,
            has_html_element: data.has_html_element,
            has_old_bundle_reference: data.has_old_bundle_reference,
            has_default_ui_reference: data.has_default_ui_reference,
            language: data.language,
            fragment: PageFragment {
                page_number: 0, // This page number is updated later once determined
                data: PageFragmentData {
                    url,
                    content,
                    filters: data.filters,
                    meta: data.meta,
                    word_count,
                    anchors,
                },
            },
            word_data,
            meta_word_data,
            sort: data.sort,
        })
    }

    fn parse_digest(
        &mut self,
        options: &SearchOptions,
    ) -> (
        String,
        HashMap<String, Vec<FossickedWord>>,
        Vec<PageAnchorData>,
        usize,
    ) {
        let mut map: HashMap<String, Vec<FossickedWord>> = HashMap::new();
        let mut anchors = Vec::new();
        // TODO: push this error handling up a level and return an Err from parse_digest
        if self.data.as_ref().is_none() {
            return ("".into(), map, anchors, 0); // empty page result, will be dropped from search
        }
        let data = self.data.as_ref().unwrap();
        let stemmer = get_stemmer(&data.language);

        let mut content = String::with_capacity(data.digest.len());

        // TODO: Consider reading newlines and jump the word_index up some amount,
        // so that separate bodies of text don't return exact string
        // matches across the boundaries. Or otherwise use some marker byte for the boundary.

        // TODO: Configure this or use segmenting across all languages

        let segment_chunks = data.digest.split_whitespace();

        #[cfg(feature = "extended")]
        let should_segment = matches!(data.language.split('-').next().unwrap(), "zh" | "ja" | "th");

        #[cfg(feature = "extended")]
        let coarse_segments = segment_chunks.map(|seg| {
            if seg.starts_with("___") {
                Either::Left(seg)
            } else {
                if should_segment {
                    // Run a segmenter only for any languages which require it.
                    Either::Right(seg.segment_str())
                } else {
                    // Currently hesitant to run segmentation during indexing
                    // that we can't also run during search, since we don't
                    // ship a segmenter to the browser. This logic is easier
                    // to replicate in the JavaScript that parses a search query.
                    Either::Left(seg)
                }
            }
        });

        #[cfg(not(feature = "extended"))]
        let coarse_segments =
            segment_chunks.map(|s| Either::<&str, core::slice::Iter<&str>>::Left(s));

        let mut total_word_index = 0;
        let mut max_word_index = 0;
        let weight_multiplier = 24.0;
        let weight_max = 10.0;
        debug_assert!(((weight_max * weight_multiplier) as u8) < std::u8::MAX);

        let mut weight_stack: Vec<u8> = vec![(1.0 * weight_multiplier) as u8];

        let mut track_word = |word: &str, append_whitespace: bool| {
            if word.chars().next() == Some('_') {
                if word.starts_with("___PAGEFIND_ANCHOR___") {
                    if let Some((element_name, anchor_id)) =
                        word.replace("___PAGEFIND_ANCHOR___", "").split_once(':')
                    {
                        let element_text = data
                            .anchor_content
                            .get(anchor_id)
                            .map(|t| normalize_content(t))
                            .unwrap_or_default();
                        let title = data.anchor_titles.get(anchor_id).cloned();

                        if let Some((_, element_id)) = anchor_id.split_once(':') {
                            anchors.push(PageAnchorData {
                                element: element_name.to_string(),
                                id: element_id.to_string(),
                                text: normalize_content(&element_text),
                                location: total_word_index as u32,
                                title,
                            });
                        }
                    }
                    return;
                }

                if word.starts_with("___PAGEFIND_WEIGHT___") {
                    let weight = word
                        .replace("___PAGEFIND_WEIGHT___", "")
                        .parse::<f32>()
                        .ok()
                        .unwrap_or(1.0);
                    if weight <= 0.0 {
                        weight_stack.push(0);
                    } else {
                        weight_stack.push(
                            (weight.clamp(0.0, weight_max).mul(weight_multiplier) as u8).max(1),
                        );
                    }
                    return;
                }

                // Auto weights are provided by the parser, and should only
                // apply if we aren't inside an explicitly weighted block,
                // in which case we should just inherit that weight.
                if word.starts_with("___PAGEFIND_AUTO_WEIGHT___") {
                    if weight_stack.len() == 1 {
                        let weight = word
                            .replace("___PAGEFIND_AUTO_WEIGHT___", "")
                            .parse::<f32>()
                            .ok()
                            .unwrap_or(1.0);
                        weight_stack
                            .push(weight.clamp(0.0, weight_max).mul(weight_multiplier) as u8);
                    } else {
                        weight_stack.push(weight_stack.last().cloned().unwrap_or_default());
                    }
                    return;
                }

                if word.starts_with("___END_PAGEFIND_WEIGHT___") {
                    weight_stack.pop();
                    return;
                }
            }

            // We use zero-width spaces as boundary values for some languages,
            // so we make sure that all are removed from the source content before going into the index.
            let base_word = word.replace('\u{200B}', "");
            if base_word.is_empty() {
                return;
            }

            content.push_str(&base_word);
            if append_whitespace {
                content.push(' ');
            }
            #[cfg(feature = "extended")]
            if should_segment {
                content.push('\u{200B}');
            }

            let word_weight = *weight_stack.last().unwrap_or(&1);

            let indexable_words =
                get_indexable_words(&base_word, stemmer.as_ref(), &options.include_characters);

            let compound_count = indexable_words
                .iter()
                .filter(|w| w.is_compound_part)
                .count();
            let partial_weight = if compound_count > 0 && word_weight > 0 {
                (word_weight / compound_count.try_into().unwrap_or(std::u8::MAX)).max(1)
            } else {
                0
            };

            for indexable in indexable_words {
                let weight = if indexable.is_compound_part {
                    partial_weight
                } else {
                    word_weight
                };

                let entry = FossickedWord {
                    position: total_word_index.try_into().unwrap(),
                    weight,
                    original_word: indexable.original,
                };
                if let Some(repeat) = map.get_mut(&indexable.stemmed) {
                    repeat.push(entry);
                } else {
                    map.insert(indexable.stemmed, vec![entry]);
                }
            }

            max_word_index = total_word_index;
            total_word_index += 1;
        };

        for segment in coarse_segments {
            match segment {
                Either::Left(word) => {
                    track_word(word, true);
                }
                Either::Right(words) => {
                    let mut words = words.peekable();
                    while let Some(word) = words.next() {
                        track_word(word, words.peek().is_none());
                    }
                }
            };
        }
        if content.ends_with('\u{200B}') {
            content.pop();
        }
        if content.ends_with(' ') {
            content.pop();
        }
        (content, map, anchors, max_word_index + 1)
    }

    /// Removes private Pagefind sentinel values from content that would otherwise leak.
    /// This should probably be handled better by not inserting these flags here in the first place,
    /// though there's a chance we do want to process them when we arrive at indexing metadata.
    fn tidy_meta_and_filters(&mut self) {
        if let Some(data) = self.data.as_mut() {
            for filter in data.filters.values_mut() {
                for filter_val in filter.iter_mut() {
                    match PRIVATE_PAGEFIND.replace_all(filter_val, "") {
                        std::borrow::Cow::Borrowed(_) => { /* no-op, no replace happened */ }
                        std::borrow::Cow::Owned(s) => *filter_val = s,
                    }
                }
            }

            for meta in data.meta.values_mut() {
                match PRIVATE_PAGEFIND.replace_all(meta, "") {
                    std::borrow::Cow::Borrowed(_) => { /* no-op, no replace happened */ }
                    std::borrow::Cow::Owned(s) => *meta = s,
                }
            }
        }
    }

    /// Parse words from metadata fields and return them with field IDs and positions.
    fn parse_meta_words(
        meta: &BTreeMap<String, String>,
        field_order: &[String],
        language: &str,
        options: &SearchOptions,
    ) -> HashMap<String, Vec<MetaFossickedWord>> {
        let mut map: HashMap<String, Vec<MetaFossickedWord>> = HashMap::new();
        let stemmer = get_stemmer(language);

        for (field_id, field_name) in field_order.iter().enumerate() {
            if let Some(field_value) = meta.get(field_name) {
                for (word_idx, word) in field_value.split_whitespace().enumerate() {
                    let indexable_words =
                        get_indexable_words(word, stemmer.as_ref(), &options.include_characters);

                    for indexable in indexable_words {
                        let entry = MetaFossickedWord {
                            field_id: field_id as u16,
                            position: word_idx as u32,
                            original_word: indexable.original,
                        };
                        map.entry(indexable.stemmed).or_default().push(entry);
                    }
                }
            }
        }

        map
    }

}

/// The URL spec's path percent-encode set, minus `/` so that the separators
/// between the path segments survive. `%`, `#` and `?` are already encoded one
/// by one today; the rest of the set covers the remaining characters that a
/// file name can carry but a URL path cannot, such as a space or a quote.
const PATH_ENCODE_SET: &AsciiSet = &CONTROLS
    .add(b' ')
    .add(b'"')
    .add(b'#')
    .add(b'%')
    .add(b'<')
    .add(b'>')
    .add(b'?')
    .add(b'`')
    .add(b'{')
    .add(b'}');

fn strip_index_html(url: &str) -> &str {
    if url.ends_with("/index.html") {
        &url[..url.len() - 10]
    } else if url == "index.html" {
        ""
    } else {
        url
    }
}

fn build_url(page_url: &Path, relative_to: Option<&Path>, options: &SearchOptions) -> String {
    let prefix = relative_to.unwrap_or(&options.site_source);

    let url = if let Ok(trimmed) = page_url.strip_prefix(prefix) {
        trimmed
    } else if page_url.is_relative() {
        page_url
    } else {
        options.logger.error(format!(
            "Absolute file was found that does not start with the source directory. Source: {:?}\nFile: {:?}",
            prefix,
            page_url
        ));
        return "/unknown/".to_string();
    };

    let final_url: String = if !options.keep_index_url {
        strip_index_html(&url.to_slash_lossy()).to_string()
    } else {
        url.to_slash_lossy().to_owned().to_string()
    };

    format!("/{}", utf8_percent_encode(&final_url, PATH_ENCODE_SET))
}

fn normalize_content(content: &str) -> String {
    let content = html_escape::decode_html_entities(content);
    let content = TRIM_NEWLINES.replace_all(&content, "");
    let content = NEWLINES.replace_all(&content, " ");
    let content = EXTRANEOUS_SPACES.replace_all(&content, " ");

    content.to_string()
}

// TODO: These language codes are duplicated with pagefind_web's Cargo.toml
fn get_stemmer(lang: &str) -> Option<Stemmer> {
    match lang.split('-').next().unwrap() {
        "ar" => Some(Stemmer::create(Algorithm::Arabic)),
        "hy" => Some(Stemmer::create(Algorithm::Armenian)),
        "eu" => Some(Stemmer::create(Algorithm::Basque)),
        "ca" => Some(Stemmer::create(Algorithm::Catalan)),
        "da" => Some(Stemmer::create(Algorithm::Danish)),
        "nl" => Some(Stemmer::create(Algorithm::Dutch)),
        "en" => Some(Stemmer::create(Algorithm::English)),
        "eo" => Some(Stemmer::create(Algorithm::Esperanto)),
        "et" => Some(Stemmer::create(Algorithm::Estonian)),
        "fi" => Some(Stemmer::create(Algorithm::Finnish)),
        "fr" => Some(Stemmer::create(Algorithm::French)),
        "de" => Some(Stemmer::create(Algorithm::German)),
        "el" => Some(Stemmer::create(Algorithm::Greek)),
        "hi" => Some(Stemmer::create(Algorithm::Hindi)),
        "hu" => Some(Stemmer::create(Algorithm::Hungarian)),
        "id" => Some(Stemmer::create(Algorithm::Indonesian)),
        "ga" => Some(Stemmer::create(Algorithm::Irish)),
        "it" => Some(Stemmer::create(Algorithm::Italian)),
        "lt" => Some(Stemmer::create(Algorithm::Lithuanian)),
        "nb" => Some(Stemmer::create(Algorithm::Norwegian)),
        "ne" => Some(Stemmer::create(Algorithm::Nepali)),
        "nn" => Some(Stemmer::create(Algorithm::Norwegian)),
        "no" => Some(Stemmer::create(Algorithm::Norwegian)),
        "pt" => Some(Stemmer::create(Algorithm::Portuguese)),
        "ro" => Some(Stemmer::create(Algorithm::Romanian)),
        "ru" => Some(Stemmer::create(Algorithm::Russian)),
        "sr" => Some(Stemmer::create(Algorithm::Serbian)),
        "pl" => Some(Stemmer::create(Algorithm::Polish)),
        "es" => Some(Stemmer::create(Algorithm::Spanish)),
        "sv" => Some(Stemmer::create(Algorithm::Swedish)),
        "ta" => Some(Stemmer::create(Algorithm::Tamil)),
        "tr" => Some(Stemmer::create(Algorithm::Turkish)),
        "yi" => Some(Stemmer::create(Algorithm::Yiddish)),
        _ => None,
    }
}

#[cfg(test)]
mod tests {
    use crate::PagefindInboundConfig;
    use twelf::Layer;

    use super::*;

    #[test]
    fn normalizing_content() {
        let input = "\nHello  Wor\n ld? \n \n";
        let output = normalize_content(input);

        assert_eq!(&output, "Hello Wor ld?");
    }

    fn test_opts() -> SearchOptions {
        std::env::set_var("PAGEFIND_SOURCE", "somewhere");
        let config =
            PagefindInboundConfig::with_layers(&[Layer::Env(Some("PAGEFIND_".into()))]).unwrap();
        SearchOptions::load(config).unwrap()
    }

    fn test_fossick(s: String) -> Fossicker {
        let mut f = Fossicker {
            file_path: Some("test/index.html".into()),
            root_path: None,
            page_url: Some("/test/".into()),
            synthetic_content: Some(s),
            data: None,
        };

        _ = f.read_synthetic_sync(&test_opts());

        f
    }

    #[tokio::test]
    async fn parse_file() {
        let mut f =
            test_fossick(["<html><body>", "<p>Hello World!</p>", "</body></html>"].concat());

        let (digest, words, _, _) = f.parse_digest(&test_opts());

        assert_eq!(digest, "Hello World!".to_string());
        assert_eq!(
            words,
            HashMap::from_iter([
                (
                    "hello".to_string(),
                    vec![FossickedWord {
                        position: 0,
                        weight: 1 * 24,
                        original_word: None,
                    }]
                ),
                (
                    "world".to_string(),
                    vec![FossickedWord {
                        position: 1,
                        weight: 1 * 24,
                        original_word: None,
                    }]
                )
            ])
        );
    }

    #[tokio::test]
    async fn parse_chars() {
        let mut f = test_fossick(
            [
                "<html><body>",
                "<p>He&amp;llo htmltag&lt;head&gt; *beföre mid*dle after*</p>",
                "</body></html>",
            ]
            .concat(),
        );

        let mut opts = test_opts();
        opts.include_characters.extend(['<', '>', '*']);
        let (digest, words, _, _) = f.parse_digest(&opts);

        assert_eq!(
            digest,
            "He&llo htmltag<head> *beföre mid*dle after*.".to_string()
        );
        assert_eq!(
            words,
            HashMap::from_iter([
                (
                    "he".to_string(),
                    vec![FossickedWord {
                        position: 0,
                        weight: 12,
                        original_word: None,
                    }]
                ),
                (
                    "llo".to_string(),
                    vec![FossickedWord {
                        position: 0,
                        weight: 12,
                        original_word: None,
                    }]
                ),
                (
                    "hello".to_string(),
                    vec![FossickedWord {
                        position: 0,
                        weight: 24,
                        original_word: None,
                    }]
                ),
                (
                    "htmltag<head>".to_string(),
                    vec![FossickedWord {
                        position: 1,
                        weight: 24,
                        original_word: None,
                    }]
                ),
                (
                    "htmltag".to_string(),
                    vec![FossickedWord {
                        position: 1,
                        weight: 12,
                        original_word: None,
                    }]
                ),
                (
                    "head".to_string(),
                    vec![FossickedWord {
                        position: 1,
                        weight: 12,
                        original_word: None,
                    }]
                ),
                (
                    "*before".to_string(),
                    vec![FossickedWord {
                        position: 2,
                        weight: 24,
                        original_word: Some("*beföre".to_string()),
                    }]
                ),
                (
                    "before".to_string(),
                    vec![FossickedWord {
                        position: 2,
                        weight: 24,
                        original_word: Some("beföre".to_string()),
                    }]
                ),
                (
                    "mid*dle".to_string(),
                    vec![FossickedWord {
                        position: 3,
                        weight: 24,
                        original_word: None,
                    }]
                ),
                (
                    "mid".to_string(),
                    vec![FossickedWord {
                        position: 3,
                        weight: 12,
                        original_word: None,
                    }]
                ),
                (
                    "dle".to_string(),
                    vec![FossickedWord {
                        position: 3,
                        weight: 12,
                        original_word: None,
                    }]
                ),
                (
                    "after*".to_string(),
                    vec![FossickedWord {
                        position: 4,
                        weight: 24,
                        original_word: None,
                    }]
                )
            ])
        );
    }

    #[tokio::test]
    async fn parse_weighted_file() {
        let mut f = test_fossick(
            [
                "<html><body>",
                "<div>The",
                "<p data-pagefind-weight='2'>Quick Brown</p>",
                "Fox",
                "<p data-pagefind-weight='0.5'>Jumps Over</p>",
                "<p data-pagefind-weight='0.00001'>Ryan</p></div>",
                "</body></html>",
            ]
            .concat(),
        );

        let (digest, words, _, _) = f.parse_digest(&test_opts());

        assert_eq!(digest, "The Quick Brown. Fox Jumps Over. Ryan.".to_string());
        assert_eq!(
            words,
            HashMap::from_iter([
                (
                    "the".to_string(),
                    vec![FossickedWord {
                        position: 0,
                        weight: 1 * 24,
                        original_word: None,
                    }]
                ),
                (
                    "quick".to_string(),
                    vec![FossickedWord {
                        position: 1,
                        weight: 2 * 24,
                        original_word: None,
                    }]
                ),
                (
                    "brown".to_string(),
                    vec![FossickedWord {
                        position: 2,
                        weight: 2 * 24,
                        original_word: None,
                    }]
                ),
                (
                    "fox".to_string(),
                    vec![FossickedWord {
                        position: 3,
                        weight: 1 * 24,
                        original_word: None,
                    }]
                ),
                (
                    "jumps".to_string(),
                    vec![FossickedWord {
                        position: 4,
                        weight: 12,
                        original_word: None,
                    }]
                ),
                (
                    "over".to_string(),
                    vec![FossickedWord {
                        position: 5,
                        weight: 12,
                        original_word: None,
                    }]
                ),
                (
                    "ryan".to_string(),
                    vec![FossickedWord {
                        position: 6,
                        weight: 1,
                        original_word: None,
                    }]
                )
            ])
        );
    }

    #[tokio::test]
    async fn parse_auto_weighted_file() {
        let mut f = test_fossick(
            [
                "<html><body>",
                "<h1>Pagefind</h1>",
                "<h2>Pagefind</h2>",
                "<h3>Pagefind</h3>",
                "<h4>Pagefind</h4>",
                "<h5>Pagefind</h5>",
                "<h6>Pagefind</h6>",
                "<p>Pagefind</p>",
                "<div data-pagefind-weight='0'><h1>Pagefind</h1></div>",
                "</body></html>",
            ]
            .concat(),
        );

        let (_, words, _, _) = f.parse_digest(&test_opts());

        assert_eq!(
            words,
            HashMap::from_iter([(
                "pagefind".to_string(),
                vec![
                    FossickedWord {
                        position: 0,
                        weight: 7 * 24,
                        original_word: None,
                    },
                    FossickedWord {
                        position: 1,
                        weight: 6 * 24,
                        original_word: None,
                    },
                    FossickedWord {
                        position: 2,
                        weight: 5 * 24,
                        original_word: None,
                    },
                    FossickedWord {
                        position: 3,
                        weight: 4 * 24,
                        original_word: None,
                    },
                    FossickedWord {
                        position: 4,
                        weight: 3 * 24,
                        original_word: None,
                    },
                    FossickedWord {
                        position: 5,
                        weight: 2 * 24,
                        original_word: None,
                    },
                    FossickedWord {
                        position: 6,
                        weight: 1 * 24,
                        original_word: None,
                    },
                    FossickedWord {
                        position: 7,
                        weight: 0 * 24,
                        original_word: None,
                    }
                ]
            )])
        );
    }

    #[tokio::test]
    async fn parse_zero_weighted_compound_words() {
        let mut f = test_fossick(
            [
                "<html><body>",
                "<div data-pagefind-weight='0'>",
                "<p>Simple text and compound.word.with.periods</p>",
                "</div>",
                "</body></html>",
            ]
            .concat(),
        );

        let (_, words, _, _) = f.parse_digest(&test_opts());

        for (_, word_positions) in words {
            for position in word_positions {
                assert_eq!(position.weight, 0, "Expected all words to have weight 0");
            }
        }
    }

    #[tokio::test]
    async fn parse_bad_weights() {
        let mut f = test_fossick(
            [
                "<html><body>",
                "<p data-pagefind-weight='lots'>The</p>",
                "<p data-pagefind-weight='99999999'>Quick</p>",
                "<p data-pagefind-weight='-1234'>Brown</p>",
                "<p data-pagefind-weight='65.4'>Fox</p>",
                "</body></html>",
            ]
            .concat(),
        );

        let (_, words, _, _) = f.parse_digest(&test_opts());

        assert_eq!(
            words,
            HashMap::from_iter([
                (
                    "the".to_string(),
                    vec![FossickedWord {
                        position: 0,
                        weight: 24,
                        original_word: None,
                    }]
                ),
                (
                    "quick".to_string(),
                    vec![FossickedWord {
                        position: 1,
                        weight: 240,
                        original_word: None,
                    }]
                ),
                (
                    "brown".to_string(),
                    vec![FossickedWord {
                        position: 2,
                        weight: 0,
                        original_word: None,
                    }]
                ),
                (
                    "fox".to_string(),
                    vec![FossickedWord {
                        position: 3,
                        weight: 240,
                        original_word: None,
                    }]
                )
            ])
        );
    }

    #[tokio::test]
    async fn parse_nbsp() {
        let mut f = test_fossick(
            [
                "<html lang='ja'><body>",
                "<p>Hello&nbsp;👋</p>",
                "</body></html>",
            ]
            .concat(),
        );

        let (_, words, _, _) = f.parse_digest(&test_opts());

        let mut words = words.keys().collect::<Vec<_>>();
        words.sort();
        assert_eq!(words, vec!["hello", "👋"]);
    }

    #[cfg(feature = "extended")]
    #[tokio::test]
    async fn parse_weights_through_segmentation() {
        let mut f = test_fossick(
            [
                "<html lang='zh'><body>",
                "<h1 id='my-title'>哎呀！ 我的错。</h1>",
                "</body></html>",
            ]
            .concat(),
        );

        let (content, words, _, _) = f.parse_digest(&test_opts());

        let mut words = words.keys().collect::<Vec<_>>();
        words.sort();
        assert_eq!(words, vec!["哎呀", "我", "的", "错"]);

        assert_eq!(
            content,
            "哎呀\u{200b}！ \u{200b}我\u{200b}的\u{200b}错\u{200b}。"
        );
    }

    #[cfg(feature = "extended")]
    #[tokio::test]
    async fn segmentation_parity_when_presplitting() {
        fn get_comparison_segmentations(full_input: &'static str) -> (Vec<String>, Vec<String>) {
            let chunked_input = full_input
                .split_whitespace()
                .filter(|w| !w.starts_with("___"))
                .collect::<Vec<_>>();
            let clean_input = chunked_input.join(" ");

            let mut legitimate_output = clean_input
                .as_str()
                .segment_str()
                .filter(|w| w.chars().any(|c| !c.is_whitespace()))
                .map(Into::into)
                .collect::<Vec<_>>();
            let mut chunked_output = chunked_input
                .into_iter()
                .flat_map(|inp| {
                    inp.segment_str()
                        .filter(|w| w.chars().any(|c| !c.is_whitespace()))
                        .collect::<Vec<_>>()
                })
                .map(Into::into)
                .collect::<Vec<_>>();

            legitimate_output.sort();
            chunked_output.sort();
            (legitimate_output, chunked_output)
        }
        {
            let full_zh_input = "___PAGEFIND_AUTO_WEIGHT___7 擁有遠端帳號權限 ___END_PAGEFIND_WEIGHT___

        我們建議大多數具有遠端帳號權限的使用者，採用 ___PAGEFIND_ANCHOR___a:0:my-link Certbot 這個 ACME 客戶端。它可以自動執行憑證的頒發、安裝，甚至不需要停止你的伺服器；Certbot 也提供專家模式，給不想要自動設定的使用者。Certbot 操作簡單，適用於許多系統；並且具有完善的文檔。參考 Certbot 官網，以獲取對於不同系統和網頁伺服器的操作說明。

        如果 Certbot 不能滿足你的需求，或是你想嘗試別的客戶端，還有很多 ACME 用戶端可供選擇。在你選定 ACME 客戶端軟體後，請參閱該客戶端的文檔。
        ___PAGEFIND_WEIGHT___44
        如果你正在嘗試使用不同的 ACME 用戶端，請使用我們的測試環境以免超過憑證頒發與更新的速率限制。
        沒有遠端帳號權限

        在沒有遠端帳號權限的情況下，最好的辦法是使用服務業者所提供的現有支援。如果你的業者支援 ___PAGEFIND_ANCHOR___a:1:my-second-link Let’s Encrypt，那麼他們就能幫助你申請免費憑證；安裝並設定自動更新。某些業者會需要你在控制介面或聯繫客服以開啟 Let’s Encrypt 服務。也有些業者會為所有客戶自動設定並安裝憑證。

        查看支援 Let’s Encrypt 的業者列表，確認你提供商的是否有出現在列表上。如果有的話，請按照他們的文檔設定 Let’s Encrypt 憑證。 ___END_PAGEFIND_WEIGHT___";

            let (legitimate_zh_output, chunked_zh_output) =
                get_comparison_segmentations(full_zh_input);
            assert_eq!(legitimate_zh_output, chunked_zh_output);
        }

        {
            let full_zh_cn_input = "没有命令行访问权限

        在没有命令行访问权限的情况下，___PAGEFIND_AUTO_WEIGHT___7 最好的办法是使用您托管服务提供商提供的内置功能。 支持 Let’s Encrypt 的服务商能替您自动完成免费证书的申请、安装、续期步骤。 某些服务商可能需要您在控制面板中开启相关选项， 也有一些服务商会自动为所有客户申请并安装证书。

        如果您的服务商存在于我们的服务商列表中， 参照其文档设置 Let’s Encrypt ___END_PAGEFIND_WEIGHT___ 证书即可。

        如果您的托管服务提供商不支持 ___PAGEFIND_ANCHOR___a:0:my-link Let’s Encrypt，您可以与他们联系请求支持。 我们尽力使添加 Let’s Encrypt 支持变得非常容易，提供商（注：非中国国内提供商）通常很乐意听取客户的建议！

        如果您的托管服务提供商不想集成 Let’s Encrypt，但支持上传自定义证书，您可以在自己的计算机上安装 Certbot 并使用手动模式（Manual Mode）。 在手动模式下，您需要将指定文件上传到您的网站以证明您的控制权。 然后，Certbot 将获取您可以上传到提供商的证书。 我们不建议使用此选项，因为它非常耗时，并且您需要在证书过期时重复此步骤。 对于大多数人来说，最好从提供商处请求 Let’s Encrypt 支持。若您的提供商不打算兼容，建议您更换提供商。
        获取帮助

        如果您对选择 ACME 客户端，使用特定客户端或与 Let’s Encrypt 相关的任何其他内容有疑问，请前往我们的社区论坛获取帮助。";

            let (legitimate_zh_cn_output, chunked_zh_cn_output) =
                get_comparison_segmentations(full_zh_cn_input);
            assert_eq!(legitimate_zh_cn_output, chunked_zh_cn_output);
        }

        {
            let full_ja_input = "___PAGEFIND_AUTO_WEIGHT___7 シェルへのアクセス権を持っている場合

            シェルアクセスができるほとんどの人には、Certbot という ACME クライアントを使うのがおすすめです。 ___END_PAGEFIND_WEIGHT___ 証明書の発行とインストールを、ダウンタイムゼロで自動化できます。 自動設定を使いたくない人のために、エキスパートモードも用意されています。 とても簡単に使え、多数のオペレーティングシステムで動作し、たくさんのドキュメントもあります。 Certbot のウェブサイトでは、各オペレーティングシステムやウェブサーバーごとの個別の設定方法について解説されています。

            Certbot があなたの要件を満たさない場合や、他のクライアントを試してみたい場合には、Certbot の他にもたくさんの ACME クライアントが利用できます。 ACME クライアントを自分で選んだ場合は、そのクライアントのドキュメントを参照してください。

            別の ACME クライアントを使って実験を行う場合は、 ___PAGEFIND_ANCHOR___a:0:my-link 私たちが用意したステージング環境を利用して、レート・リミットの制限を受けないように気をつけてください。
            シェルへのアクセス権を持っていない場合

            シェルアクセスができない場合に Let’s Encrypt を利用する一番良い方法は、ホスティング・プロバイダが用意したサポートを利用することです。 もし、あなたが利用するホスティング・プロバイダが Let’s Encrypt をサポートしている場合、あなたの代わりに無料の証明書をリクエスト、インストールし、自動的に最新の状態に更新してくれます。 一部のホスティング・プロバイダでは、この機能は自分で設定から有効にする必要がある場合があります。 それ以外のプロバイダでは、すべてのユーザーのために、自動で証明書が発行・インストールされるようになっています。

            あなたが利用しているホスティング・プロバイダが Let’s Encrypt をサポートしているかどうかは、 ホスティング・プロバイダのリストで確認してください。 もしサポートされている場合は、ホスティング・プロバイダのドキュメンに書かれている Let’s Encrypt の設定方法に従ってください。";

            let (legitimate_ja_output, chunked_ja_output) =
                get_comparison_segmentations(full_ja_input);
            assert_eq!(legitimate_ja_output, chunked_ja_output);
        }
    }

    #[cfg(not(target_os = "windows"))]
    #[test]
    fn building_url() {
        std::env::set_var("PAGEFIND_SITE", "hello/world");
        let config =
            PagefindInboundConfig::with_layers(&[Layer::Env(Some("PAGEFIND_".into()))]).unwrap();
        let opts = SearchOptions::load(config).unwrap();

        let cwd = std::env::current_dir().unwrap();

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/index.html".into());
        assert_eq!(&build_url(&p, None, &opts), "/");

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/about/index.html".into());
        assert_eq!(&build_url(&p, None, &opts), "/about/");

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/about.html".into());
        assert_eq!(&build_url(&p, None, &opts), "/about.html");

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/about/index.htm".into());
        assert_eq!(&build_url(&p, None, &opts), "/about/index.htm");

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/index.html".into());
        let root: PathBuf = cwd.join::<PathBuf>("hello".into());
        assert_eq!(&build_url(&p, Some(&root), &opts), "/world/");

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/100%/C#/why?.html".into());
        assert_eq!(&build_url(&p, None, &opts), "/100%25/C%23/why%3F.html");
    }

    #[cfg(not(target_os = "windows"))]
    #[test]
    fn building_urls_with_characters_that_need_encoding() {
        std::env::set_var("PAGEFIND_SITE", "hello/world");
        let config =
            PagefindInboundConfig::with_layers(&[Layer::Env(Some("PAGEFIND_".into()))]).unwrap();
        let opts = SearchOptions::load(config).unwrap();

        let cwd = std::env::current_dir().unwrap();

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/Müsli.html".into());
        assert_eq!(&build_url(&p, None, &opts), "/M%C3%BCsli.html");

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/Müsli/index.html".into());
        assert_eq!(&build_url(&p, None, &opts), "/M%C3%BCsli/");

        let p: PathBuf = cwd.join::<PathBuf>("hello/world/a b/c#d?e.html".into());
        assert_eq!(&build_url(&p, None, &opts), "/a%20b/c%23d%3Fe.html");

        // A literal percent in the file name is encoded, so the URL points back at
        // the file that is on disk rather than at the one it looks like it names.
        let p: PathBuf = cwd.join::<PathBuf>("hello/world/M%C3%BCsli.html".into());
        assert_eq!(&build_url(&p, None, &opts), "/M%25C3%25BCsli.html");
    }

    #[cfg(target_os = "windows")]
    #[test]
    fn building_windows_urls() {
        std::env::set_var("PAGEFIND_SITE", "C:\\hello\\world");
        let config =
            PagefindInboundConfig::with_layers(&[Layer::Env(Some("PAGEFIND_".into()))]).unwrap();
        let opts = SearchOptions::load(config).unwrap();

        let p: PathBuf = "C:\\hello\\world\\index.html".into();
        assert_eq!(&build_url(&p, None, &opts), "/");

        let p: PathBuf = "C:\\hello\\world\\about\\index.html".into();
        assert_eq!(&build_url(&p, None, &opts), "/about/");

        let p: PathBuf = "C:\\hello\\world\\about\\index.htm".into();
        assert_eq!(&build_url(&p, None, &opts), "/about/index.htm");
    }
}