dualfroz
49fb9b1b9b
build_url() escaped %, # and ? one by one, so a file name carrying any other character that a URL path cannot hold - a space, a quote, an angle bracket, a control character - was emitted raw and only resolved if the host happened to serve the same byte sequence the index was built from. Encode the URL that build_url derives from the file path with the URL spec's path percent-encode set, leaving / alone so the separators between path segments survive. % stays in the set, so a file literally named in encoded form keeps pointing at the file that is on disk. URLs supplied by the caller (data-pagefind-meta="url:...", addCustomRecord) do not go through build_url and are untouched. Fixes #619
1313 lines
50 KiB
Rust
1313 lines
50 KiB
Rust
use anyhow::{bail, Result};
|
||
#[cfg(feature = "extended")]
|
||
use charabia::Segment;
|
||
use either::Either;
|
||
use flate2::read::GzDecoder;
|
||
use hashbrown::HashMap;
|
||
use lazy_static::lazy_static;
|
||
use pagefind_stem::{Algorithm, Stemmer};
|
||
use path_slash::PathExt as _;
|
||
use percent_encoding::{utf8_percent_encode, AsciiSet, CONTROLS};
|
||
use regex::Regex;
|
||
use std::collections::BTreeMap;
|
||
use std::io::BufRead;
|
||
use std::io::Error;
|
||
use std::io::Read;
|
||
use std::ops::Mul;
|
||
use std::path::{Path, PathBuf};
|
||
|
||
use crate::fossick::splitting::get_indexable_words;
|
||
use crate::fragments::{PageAnchorData, PageFragment, PageFragmentData};
|
||
use crate::SearchOptions;
|
||
use parser::DomParser;
|
||
|
||
use self::parser::DomParserResult;
|
||
|
||
lazy_static! {
|
||
static ref NEWLINES: Regex = Regex::new("(\n|\r\n)+").unwrap();
|
||
static ref TRIM_NEWLINES: Regex = Regex::new("^[\n\r\\s]+|[\n\r\\s]+$").unwrap();
|
||
static ref EXTRANEOUS_SPACES: Regex = Regex::new("\\s{2,}").unwrap();
|
||
static ref PRIVATE_PAGEFIND: Regex = Regex::new("___PAGEFIND_[\\S]+\\s?").unwrap();
|
||
}
|
||
|
||
pub mod parser;
|
||
mod splitting;
|
||
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub struct FossickedWord {
|
||
pub position: u32,
|
||
pub weight: u8,
|
||
/// The original word before diacritic normalization, if it differs from the normalized form.
|
||
pub original_word: Option<String>,
|
||
}
|
||
|
||
#[derive(Debug, Clone, PartialEq)]
|
||
pub struct MetaFossickedWord {
|
||
pub field_id: u16,
|
||
pub position: u32,
|
||
/// The original word before diacritic normalization, if it differs from the normalized form.
|
||
pub original_word: Option<String>,
|
||
}
|
||
|
||
#[derive(Debug, Clone)]
|
||
pub struct FossickedData {
|
||
pub url: String,
|
||
pub fragment: PageFragment,
|
||
pub word_data: HashMap<String, Vec<FossickedWord>>,
|
||
pub meta_word_data: HashMap<String, Vec<MetaFossickedWord>>,
|
||
pub sort: BTreeMap<String, String>,
|
||
pub has_custom_body: bool,
|
||
pub force_inclusion: bool,
|
||
pub has_html_element: bool,
|
||
pub has_old_bundle_reference: bool,
|
||
pub has_default_ui_reference: bool,
|
||
pub language: String,
|
||
}
|
||
|
||
#[derive(Debug)]
|
||
pub struct Fossicker {
|
||
file_path: Option<PathBuf>,
|
||
/// Built URLs should be relative to this directory
|
||
root_path: Option<PathBuf>,
|
||
page_url: Option<String>,
|
||
synthetic_content: Option<String>,
|
||
data: Option<DomParserResult>,
|
||
}
|
||
|
||
impl Fossicker {
|
||
pub fn new_relative_to(file_path: PathBuf, root_path: PathBuf) -> Self {
|
||
Self {
|
||
file_path: Some(file_path),
|
||
root_path: Some(root_path),
|
||
page_url: None,
|
||
synthetic_content: None,
|
||
data: None,
|
||
}
|
||
}
|
||
|
||
pub fn new_synthetic(
|
||
file_path: Option<PathBuf>,
|
||
page_url: Option<String>,
|
||
contents: String,
|
||
) -> Self {
|
||
Self {
|
||
file_path,
|
||
root_path: None,
|
||
page_url,
|
||
synthetic_content: Some(contents),
|
||
data: None,
|
||
}
|
||
}
|
||
|
||
pub fn new_with_data(url: String, data: DomParserResult) -> Self {
|
||
Self {
|
||
file_path: None,
|
||
root_path: None,
|
||
page_url: Some(url),
|
||
synthetic_content: None,
|
||
data: Some(data),
|
||
}
|
||
}
|
||
|
||
fn read_file_sync(&mut self, options: &SearchOptions) -> Result<(), Error> {
|
||
let Some(file_path) = &self.file_path else {
|
||
return Ok(());
|
||
};
|
||
let file = std::fs::File::open(file_path)?;
|
||
|
||
let mut rewriter = DomParser::new(options);
|
||
|
||
let mut br = std::io::BufReader::new(file);
|
||
let mut buf = [0; 20000];
|
||
|
||
// Check for gzip magic bytes
|
||
let is_gzip = {
|
||
let peek = br.fill_buf()?;
|
||
peek.len() >= 3 && peek[0] == 0x1F && peek[1] == 0x8B && peek[2] == 0x08
|
||
};
|
||
|
||
if is_gzip {
|
||
let mut decoder = GzDecoder::new(br);
|
||
loop {
|
||
match decoder.read(&mut buf) {
|
||
Ok(0) => break,
|
||
Ok(read) => {
|
||
if let Err(error) = rewriter.write(&buf[..read]) {
|
||
options.logger.error(format!(
|
||
"Failed to parse file {} — skipping this file. Error:\n{error}",
|
||
file_path.to_str().unwrap_or("[unknown file]"),
|
||
));
|
||
return Ok(());
|
||
}
|
||
}
|
||
Err(e) => {
|
||
options.logger.error(format!(
|
||
"IO error reading gzip file {}: {e}",
|
||
file_path.to_str().unwrap_or("[unknown file]")
|
||
));
|
||
return Err(e);
|
||
}
|
||
}
|
||
}
|
||
} else {
|
||
loop {
|
||
match br.read(&mut buf) {
|
||
Ok(0) => break,
|
||
Ok(read) => {
|
||
if let Err(error) = rewriter.write(&buf[..read]) {
|
||
options.logger.error(format!(
|
||
"Failed to parse file {} — skipping this file. Error:\n{error}",
|
||
file_path.to_str().unwrap_or("[unknown file]")
|
||
));
|
||
return Ok(());
|
||
}
|
||
}
|
||
Err(e) => {
|
||
options.logger.error(format!(
|
||
"IO error reading file {}: {e}",
|
||
file_path.to_str().unwrap_or("[unknown file]")
|
||
));
|
||
return Err(e);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
let mut data = rewriter.wrap();
|
||
if let Some(forced_language) = &options.force_language {
|
||
data.language = forced_language.clone();
|
||
}
|
||
|
||
self.data = Some(data);
|
||
|
||
Ok(())
|
||
}
|
||
|
||
fn read_synthetic_sync(&mut self, options: &SearchOptions) -> Result<(), Error> {
|
||
let Some(contents) = self.synthetic_content.as_ref() else {
|
||
return Ok(());
|
||
};
|
||
|
||
let mut rewriter = DomParser::new(options);
|
||
|
||
let mut br = std::io::Cursor::new(contents.as_bytes());
|
||
let mut buf = [0; 20000];
|
||
|
||
loop {
|
||
match Read::read(&mut br, &mut buf) {
|
||
Ok(0) => break,
|
||
Ok(read) => {
|
||
if let Err(error) = rewriter.write(&buf[..read]) {
|
||
let path_desc = self
|
||
.file_path
|
||
.as_ref()
|
||
.and_then(|p| p.to_str())
|
||
.or(self.page_url.as_deref())
|
||
.unwrap_or("[unknown file]");
|
||
options.logger.error(format!(
|
||
"Failed to parse file {path_desc} — skipping this file. Error:\n{error}"
|
||
));
|
||
return Ok(());
|
||
}
|
||
}
|
||
Err(e) => {
|
||
let path_desc = self
|
||
.file_path
|
||
.as_ref()
|
||
.and_then(|p| p.to_str())
|
||
.or(self.page_url.as_deref())
|
||
.unwrap_or("[unknown file]");
|
||
options.logger.error(format!(
|
||
"IO error reading synthetic content for {path_desc}: {e}"
|
||
));
|
||
return Err(e);
|
||
}
|
||
}
|
||
}
|
||
|
||
let mut data = rewriter.wrap();
|
||
if let Some(forced_language) = &options.force_language {
|
||
data.language = forced_language.clone();
|
||
}
|
||
|
||
self.data = Some(data);
|
||
|
||
Ok(())
|
||
}
|
||
|
||
/// Retries up to MAX_RETRIES times with exponential backoff on transient IO errors.
|
||
fn fossick_html_sync(&mut self, options: &SearchOptions) -> Result<(), std::io::Error> {
|
||
const MAX_RETRIES: u32 = 10;
|
||
|
||
let mut last_error = None;
|
||
for attempt in 0..MAX_RETRIES {
|
||
let result = if self.synthetic_content.is_some() {
|
||
self.read_synthetic_sync(options)
|
||
} else {
|
||
self.read_file_sync(options)
|
||
};
|
||
|
||
match result {
|
||
Ok(()) => return Ok(()),
|
||
Err(e) => {
|
||
last_error = Some(e);
|
||
if attempt < MAX_RETRIES - 1 {
|
||
// Exponential backoff: 1ms, 2ms, ... 512ms
|
||
std::thread::sleep(std::time::Duration::from_millis(1 << attempt));
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
Err(last_error.unwrap_or_else(|| {
|
||
std::io::Error::new(std::io::ErrorKind::Other, "Max retries exceeded")
|
||
}))
|
||
}
|
||
|
||
pub fn fossick_sync(mut self, options: &SearchOptions) -> Result<FossickedData> {
|
||
if (self.file_path.is_some() || self.synthetic_content.is_some()) && self.data.is_none() {
|
||
if let Err(e) = self.fossick_html_sync(options) {
|
||
let path_desc = self
|
||
.file_path
|
||
.as_ref()
|
||
.and_then(|p| p.to_str())
|
||
.or(self.page_url.as_deref())
|
||
.unwrap_or("[unknown file]");
|
||
options
|
||
.logger
|
||
.error(format!("Failed to read {path_desc} after retries: {e}"));
|
||
bail!("Failed to read {path_desc}: {e}");
|
||
}
|
||
}
|
||
|
||
let (content, word_data, anchors, word_count) = self.parse_digest(options);
|
||
self.tidy_meta_and_filters();
|
||
|
||
let data = self.data.unwrap();
|
||
|
||
// Get sorted list of meta field names for consistent field IDs
|
||
let meta_field_order: Vec<String> = data.meta.keys().cloned().collect();
|
||
let meta_word_data =
|
||
Self::parse_meta_words(&data.meta, &meta_field_order, &data.language, options);
|
||
|
||
// Build URL using Option combinators for cleaner logic
|
||
let url = self
|
||
.page_url
|
||
.clone()
|
||
.or_else(|| {
|
||
self.file_path
|
||
.as_ref()
|
||
.map(|path| build_url(path, self.root_path.as_deref(), options))
|
||
})
|
||
.ok_or_else(|| {
|
||
options
|
||
.logger
|
||
.error("Tried to index file with no specified URL or file path, ignoring.");
|
||
anyhow::anyhow!("Tried to index file with no specified URL or file path, ignoring.")
|
||
})?;
|
||
|
||
Ok(FossickedData {
|
||
url: url.clone(), // Clone needed since url is used in both struct and fragment.data
|
||
has_custom_body: data.has_custom_body,
|
||
force_inclusion: data.force_inclusion,
|
||
has_html_element: data.has_html_element,
|
||
has_old_bundle_reference: data.has_old_bundle_reference,
|
||
has_default_ui_reference: data.has_default_ui_reference,
|
||
language: data.language,
|
||
fragment: PageFragment {
|
||
page_number: 0, // This page number is updated later once determined
|
||
data: PageFragmentData {
|
||
url,
|
||
content,
|
||
filters: data.filters,
|
||
meta: data.meta,
|
||
word_count,
|
||
anchors,
|
||
},
|
||
},
|
||
word_data,
|
||
meta_word_data,
|
||
sort: data.sort,
|
||
})
|
||
}
|
||
|
||
fn parse_digest(
|
||
&mut self,
|
||
options: &SearchOptions,
|
||
) -> (
|
||
String,
|
||
HashMap<String, Vec<FossickedWord>>,
|
||
Vec<PageAnchorData>,
|
||
usize,
|
||
) {
|
||
let mut map: HashMap<String, Vec<FossickedWord>> = HashMap::new();
|
||
let mut anchors = Vec::new();
|
||
// TODO: push this error handling up a level and return an Err from parse_digest
|
||
if self.data.as_ref().is_none() {
|
||
return ("".into(), map, anchors, 0); // empty page result, will be dropped from search
|
||
}
|
||
let data = self.data.as_ref().unwrap();
|
||
let stemmer = get_stemmer(&data.language);
|
||
|
||
let mut content = String::with_capacity(data.digest.len());
|
||
|
||
// TODO: Consider reading newlines and jump the word_index up some amount,
|
||
// so that separate bodies of text don't return exact string
|
||
// matches across the boundaries. Or otherwise use some marker byte for the boundary.
|
||
|
||
// TODO: Configure this or use segmenting across all languages
|
||
|
||
let segment_chunks = data.digest.split_whitespace();
|
||
|
||
#[cfg(feature = "extended")]
|
||
let should_segment = matches!(data.language.split('-').next().unwrap(), "zh" | "ja" | "th");
|
||
|
||
#[cfg(feature = "extended")]
|
||
let coarse_segments = segment_chunks.map(|seg| {
|
||
if seg.starts_with("___") {
|
||
Either::Left(seg)
|
||
} else {
|
||
if should_segment {
|
||
// Run a segmenter only for any languages which require it.
|
||
Either::Right(seg.segment_str())
|
||
} else {
|
||
// Currently hesitant to run segmentation during indexing
|
||
// that we can't also run during search, since we don't
|
||
// ship a segmenter to the browser. This logic is easier
|
||
// to replicate in the JavaScript that parses a search query.
|
||
Either::Left(seg)
|
||
}
|
||
}
|
||
});
|
||
|
||
#[cfg(not(feature = "extended"))]
|
||
let coarse_segments =
|
||
segment_chunks.map(|s| Either::<&str, core::slice::Iter<&str>>::Left(s));
|
||
|
||
let mut total_word_index = 0;
|
||
let mut max_word_index = 0;
|
||
let weight_multiplier = 24.0;
|
||
let weight_max = 10.0;
|
||
debug_assert!(((weight_max * weight_multiplier) as u8) < std::u8::MAX);
|
||
|
||
let mut weight_stack: Vec<u8> = vec![(1.0 * weight_multiplier) as u8];
|
||
|
||
let mut track_word = |word: &str, append_whitespace: bool| {
|
||
if word.chars().next() == Some('_') {
|
||
if word.starts_with("___PAGEFIND_ANCHOR___") {
|
||
if let Some((element_name, anchor_id)) =
|
||
word.replace("___PAGEFIND_ANCHOR___", "").split_once(':')
|
||
{
|
||
let element_text = data
|
||
.anchor_content
|
||
.get(anchor_id)
|
||
.map(|t| normalize_content(t))
|
||
.unwrap_or_default();
|
||
let title = data.anchor_titles.get(anchor_id).cloned();
|
||
|
||
if let Some((_, element_id)) = anchor_id.split_once(':') {
|
||
anchors.push(PageAnchorData {
|
||
element: element_name.to_string(),
|
||
id: element_id.to_string(),
|
||
text: normalize_content(&element_text),
|
||
location: total_word_index as u32,
|
||
title,
|
||
});
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
|
||
if word.starts_with("___PAGEFIND_WEIGHT___") {
|
||
let weight = word
|
||
.replace("___PAGEFIND_WEIGHT___", "")
|
||
.parse::<f32>()
|
||
.ok()
|
||
.unwrap_or(1.0);
|
||
if weight <= 0.0 {
|
||
weight_stack.push(0);
|
||
} else {
|
||
weight_stack.push(
|
||
(weight.clamp(0.0, weight_max).mul(weight_multiplier) as u8).max(1),
|
||
);
|
||
}
|
||
return;
|
||
}
|
||
|
||
// Auto weights are provided by the parser, and should only
|
||
// apply if we aren't inside an explicitly weighted block,
|
||
// in which case we should just inherit that weight.
|
||
if word.starts_with("___PAGEFIND_AUTO_WEIGHT___") {
|
||
if weight_stack.len() == 1 {
|
||
let weight = word
|
||
.replace("___PAGEFIND_AUTO_WEIGHT___", "")
|
||
.parse::<f32>()
|
||
.ok()
|
||
.unwrap_or(1.0);
|
||
weight_stack
|
||
.push(weight.clamp(0.0, weight_max).mul(weight_multiplier) as u8);
|
||
} else {
|
||
weight_stack.push(weight_stack.last().cloned().unwrap_or_default());
|
||
}
|
||
return;
|
||
}
|
||
|
||
if word.starts_with("___END_PAGEFIND_WEIGHT___") {
|
||
weight_stack.pop();
|
||
return;
|
||
}
|
||
}
|
||
|
||
// We use zero-width spaces as boundary values for some languages,
|
||
// so we make sure that all are removed from the source content before going into the index.
|
||
let base_word = word.replace('\u{200B}', "");
|
||
if base_word.is_empty() {
|
||
return;
|
||
}
|
||
|
||
content.push_str(&base_word);
|
||
if append_whitespace {
|
||
content.push(' ');
|
||
}
|
||
#[cfg(feature = "extended")]
|
||
if should_segment {
|
||
content.push('\u{200B}');
|
||
}
|
||
|
||
let word_weight = *weight_stack.last().unwrap_or(&1);
|
||
|
||
let indexable_words =
|
||
get_indexable_words(&base_word, stemmer.as_ref(), &options.include_characters);
|
||
|
||
let compound_count = indexable_words
|
||
.iter()
|
||
.filter(|w| w.is_compound_part)
|
||
.count();
|
||
let partial_weight = if compound_count > 0 && word_weight > 0 {
|
||
(word_weight / compound_count.try_into().unwrap_or(std::u8::MAX)).max(1)
|
||
} else {
|
||
0
|
||
};
|
||
|
||
for indexable in indexable_words {
|
||
let weight = if indexable.is_compound_part {
|
||
partial_weight
|
||
} else {
|
||
word_weight
|
||
};
|
||
|
||
let entry = FossickedWord {
|
||
position: total_word_index.try_into().unwrap(),
|
||
weight,
|
||
original_word: indexable.original,
|
||
};
|
||
if let Some(repeat) = map.get_mut(&indexable.stemmed) {
|
||
repeat.push(entry);
|
||
} else {
|
||
map.insert(indexable.stemmed, vec![entry]);
|
||
}
|
||
}
|
||
|
||
max_word_index = total_word_index;
|
||
total_word_index += 1;
|
||
};
|
||
|
||
for segment in coarse_segments {
|
||
match segment {
|
||
Either::Left(word) => {
|
||
track_word(word, true);
|
||
}
|
||
Either::Right(words) => {
|
||
let mut words = words.peekable();
|
||
while let Some(word) = words.next() {
|
||
track_word(word, words.peek().is_none());
|
||
}
|
||
}
|
||
};
|
||
}
|
||
if content.ends_with('\u{200B}') {
|
||
content.pop();
|
||
}
|
||
if content.ends_with(' ') {
|
||
content.pop();
|
||
}
|
||
(content, map, anchors, max_word_index + 1)
|
||
}
|
||
|
||
/// Removes private Pagefind sentinel values from content that would otherwise leak.
|
||
/// This should probably be handled better by not inserting these flags here in the first place,
|
||
/// though there's a chance we do want to process them when we arrive at indexing metadata.
|
||
fn tidy_meta_and_filters(&mut self) {
|
||
if let Some(data) = self.data.as_mut() {
|
||
for filter in data.filters.values_mut() {
|
||
for filter_val in filter.iter_mut() {
|
||
match PRIVATE_PAGEFIND.replace_all(filter_val, "") {
|
||
std::borrow::Cow::Borrowed(_) => { /* no-op, no replace happened */ }
|
||
std::borrow::Cow::Owned(s) => *filter_val = s,
|
||
}
|
||
}
|
||
}
|
||
|
||
for meta in data.meta.values_mut() {
|
||
match PRIVATE_PAGEFIND.replace_all(meta, "") {
|
||
std::borrow::Cow::Borrowed(_) => { /* no-op, no replace happened */ }
|
||
std::borrow::Cow::Owned(s) => *meta = s,
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Parse words from metadata fields and return them with field IDs and positions.
|
||
fn parse_meta_words(
|
||
meta: &BTreeMap<String, String>,
|
||
field_order: &[String],
|
||
language: &str,
|
||
options: &SearchOptions,
|
||
) -> HashMap<String, Vec<MetaFossickedWord>> {
|
||
let mut map: HashMap<String, Vec<MetaFossickedWord>> = HashMap::new();
|
||
let stemmer = get_stemmer(language);
|
||
|
||
for (field_id, field_name) in field_order.iter().enumerate() {
|
||
if let Some(field_value) = meta.get(field_name) {
|
||
for (word_idx, word) in field_value.split_whitespace().enumerate() {
|
||
let indexable_words =
|
||
get_indexable_words(word, stemmer.as_ref(), &options.include_characters);
|
||
|
||
for indexable in indexable_words {
|
||
let entry = MetaFossickedWord {
|
||
field_id: field_id as u16,
|
||
position: word_idx as u32,
|
||
original_word: indexable.original,
|
||
};
|
||
map.entry(indexable.stemmed).or_default().push(entry);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
map
|
||
}
|
||
|
||
}
|
||
|
||
/// The URL spec's path percent-encode set, minus `/` so that the separators
|
||
/// between the path segments survive. `%`, `#` and `?` are already encoded one
|
||
/// by one today; the rest of the set covers the remaining characters that a
|
||
/// file name can carry but a URL path cannot, such as a space or a quote.
|
||
const PATH_ENCODE_SET: &AsciiSet = &CONTROLS
|
||
.add(b' ')
|
||
.add(b'"')
|
||
.add(b'#')
|
||
.add(b'%')
|
||
.add(b'<')
|
||
.add(b'>')
|
||
.add(b'?')
|
||
.add(b'`')
|
||
.add(b'{')
|
||
.add(b'}');
|
||
|
||
fn strip_index_html(url: &str) -> &str {
|
||
if url.ends_with("/index.html") {
|
||
&url[..url.len() - 10]
|
||
} else if url == "index.html" {
|
||
""
|
||
} else {
|
||
url
|
||
}
|
||
}
|
||
|
||
fn build_url(page_url: &Path, relative_to: Option<&Path>, options: &SearchOptions) -> String {
|
||
let prefix = relative_to.unwrap_or(&options.site_source);
|
||
|
||
let url = if let Ok(trimmed) = page_url.strip_prefix(prefix) {
|
||
trimmed
|
||
} else if page_url.is_relative() {
|
||
page_url
|
||
} else {
|
||
options.logger.error(format!(
|
||
"Absolute file was found that does not start with the source directory. Source: {:?}\nFile: {:?}",
|
||
prefix,
|
||
page_url
|
||
));
|
||
return "/unknown/".to_string();
|
||
};
|
||
|
||
let final_url: String = if !options.keep_index_url {
|
||
strip_index_html(&url.to_slash_lossy()).to_string()
|
||
} else {
|
||
url.to_slash_lossy().to_owned().to_string()
|
||
};
|
||
|
||
format!("/{}", utf8_percent_encode(&final_url, PATH_ENCODE_SET))
|
||
}
|
||
|
||
fn normalize_content(content: &str) -> String {
|
||
let content = html_escape::decode_html_entities(content);
|
||
let content = TRIM_NEWLINES.replace_all(&content, "");
|
||
let content = NEWLINES.replace_all(&content, " ");
|
||
let content = EXTRANEOUS_SPACES.replace_all(&content, " ");
|
||
|
||
content.to_string()
|
||
}
|
||
|
||
// TODO: These language codes are duplicated with pagefind_web's Cargo.toml
|
||
fn get_stemmer(lang: &str) -> Option<Stemmer> {
|
||
match lang.split('-').next().unwrap() {
|
||
"ar" => Some(Stemmer::create(Algorithm::Arabic)),
|
||
"hy" => Some(Stemmer::create(Algorithm::Armenian)),
|
||
"eu" => Some(Stemmer::create(Algorithm::Basque)),
|
||
"ca" => Some(Stemmer::create(Algorithm::Catalan)),
|
||
"da" => Some(Stemmer::create(Algorithm::Danish)),
|
||
"nl" => Some(Stemmer::create(Algorithm::Dutch)),
|
||
"en" => Some(Stemmer::create(Algorithm::English)),
|
||
"eo" => Some(Stemmer::create(Algorithm::Esperanto)),
|
||
"et" => Some(Stemmer::create(Algorithm::Estonian)),
|
||
"fi" => Some(Stemmer::create(Algorithm::Finnish)),
|
||
"fr" => Some(Stemmer::create(Algorithm::French)),
|
||
"de" => Some(Stemmer::create(Algorithm::German)),
|
||
"el" => Some(Stemmer::create(Algorithm::Greek)),
|
||
"hi" => Some(Stemmer::create(Algorithm::Hindi)),
|
||
"hu" => Some(Stemmer::create(Algorithm::Hungarian)),
|
||
"id" => Some(Stemmer::create(Algorithm::Indonesian)),
|
||
"ga" => Some(Stemmer::create(Algorithm::Irish)),
|
||
"it" => Some(Stemmer::create(Algorithm::Italian)),
|
||
"lt" => Some(Stemmer::create(Algorithm::Lithuanian)),
|
||
"nb" => Some(Stemmer::create(Algorithm::Norwegian)),
|
||
"ne" => Some(Stemmer::create(Algorithm::Nepali)),
|
||
"nn" => Some(Stemmer::create(Algorithm::Norwegian)),
|
||
"no" => Some(Stemmer::create(Algorithm::Norwegian)),
|
||
"pt" => Some(Stemmer::create(Algorithm::Portuguese)),
|
||
"ro" => Some(Stemmer::create(Algorithm::Romanian)),
|
||
"ru" => Some(Stemmer::create(Algorithm::Russian)),
|
||
"sr" => Some(Stemmer::create(Algorithm::Serbian)),
|
||
"pl" => Some(Stemmer::create(Algorithm::Polish)),
|
||
"es" => Some(Stemmer::create(Algorithm::Spanish)),
|
||
"sv" => Some(Stemmer::create(Algorithm::Swedish)),
|
||
"ta" => Some(Stemmer::create(Algorithm::Tamil)),
|
||
"tr" => Some(Stemmer::create(Algorithm::Turkish)),
|
||
"yi" => Some(Stemmer::create(Algorithm::Yiddish)),
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use crate::PagefindInboundConfig;
|
||
use twelf::Layer;
|
||
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn normalizing_content() {
|
||
let input = "\nHello Wor\n ld? \n \n";
|
||
let output = normalize_content(input);
|
||
|
||
assert_eq!(&output, "Hello Wor ld?");
|
||
}
|
||
|
||
fn test_opts() -> SearchOptions {
|
||
std::env::set_var("PAGEFIND_SOURCE", "somewhere");
|
||
let config =
|
||
PagefindInboundConfig::with_layers(&[Layer::Env(Some("PAGEFIND_".into()))]).unwrap();
|
||
SearchOptions::load(config).unwrap()
|
||
}
|
||
|
||
fn test_fossick(s: String) -> Fossicker {
|
||
let mut f = Fossicker {
|
||
file_path: Some("test/index.html".into()),
|
||
root_path: None,
|
||
page_url: Some("/test/".into()),
|
||
synthetic_content: Some(s),
|
||
data: None,
|
||
};
|
||
|
||
_ = f.read_synthetic_sync(&test_opts());
|
||
|
||
f
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn parse_file() {
|
||
let mut f =
|
||
test_fossick(["<html><body>", "<p>Hello World!</p>", "</body></html>"].concat());
|
||
|
||
let (digest, words, _, _) = f.parse_digest(&test_opts());
|
||
|
||
assert_eq!(digest, "Hello World!".to_string());
|
||
assert_eq!(
|
||
words,
|
||
HashMap::from_iter([
|
||
(
|
||
"hello".to_string(),
|
||
vec![FossickedWord {
|
||
position: 0,
|
||
weight: 1 * 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"world".to_string(),
|
||
vec![FossickedWord {
|
||
position: 1,
|
||
weight: 1 * 24,
|
||
original_word: None,
|
||
}]
|
||
)
|
||
])
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn parse_chars() {
|
||
let mut f = test_fossick(
|
||
[
|
||
"<html><body>",
|
||
"<p>He&llo htmltag<head> *beföre mid*dle after*</p>",
|
||
"</body></html>",
|
||
]
|
||
.concat(),
|
||
);
|
||
|
||
let mut opts = test_opts();
|
||
opts.include_characters.extend(['<', '>', '*']);
|
||
let (digest, words, _, _) = f.parse_digest(&opts);
|
||
|
||
assert_eq!(
|
||
digest,
|
||
"He&llo htmltag<head> *beföre mid*dle after*.".to_string()
|
||
);
|
||
assert_eq!(
|
||
words,
|
||
HashMap::from_iter([
|
||
(
|
||
"he".to_string(),
|
||
vec![FossickedWord {
|
||
position: 0,
|
||
weight: 12,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"llo".to_string(),
|
||
vec![FossickedWord {
|
||
position: 0,
|
||
weight: 12,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"hello".to_string(),
|
||
vec![FossickedWord {
|
||
position: 0,
|
||
weight: 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"htmltag<head>".to_string(),
|
||
vec![FossickedWord {
|
||
position: 1,
|
||
weight: 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"htmltag".to_string(),
|
||
vec![FossickedWord {
|
||
position: 1,
|
||
weight: 12,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"head".to_string(),
|
||
vec![FossickedWord {
|
||
position: 1,
|
||
weight: 12,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"*before".to_string(),
|
||
vec![FossickedWord {
|
||
position: 2,
|
||
weight: 24,
|
||
original_word: Some("*beföre".to_string()),
|
||
}]
|
||
),
|
||
(
|
||
"before".to_string(),
|
||
vec![FossickedWord {
|
||
position: 2,
|
||
weight: 24,
|
||
original_word: Some("beföre".to_string()),
|
||
}]
|
||
),
|
||
(
|
||
"mid*dle".to_string(),
|
||
vec![FossickedWord {
|
||
position: 3,
|
||
weight: 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"mid".to_string(),
|
||
vec![FossickedWord {
|
||
position: 3,
|
||
weight: 12,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"dle".to_string(),
|
||
vec![FossickedWord {
|
||
position: 3,
|
||
weight: 12,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"after*".to_string(),
|
||
vec![FossickedWord {
|
||
position: 4,
|
||
weight: 24,
|
||
original_word: None,
|
||
}]
|
||
)
|
||
])
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn parse_weighted_file() {
|
||
let mut f = test_fossick(
|
||
[
|
||
"<html><body>",
|
||
"<div>The",
|
||
"<p data-pagefind-weight='2'>Quick Brown</p>",
|
||
"Fox",
|
||
"<p data-pagefind-weight='0.5'>Jumps Over</p>",
|
||
"<p data-pagefind-weight='0.00001'>Ryan</p></div>",
|
||
"</body></html>",
|
||
]
|
||
.concat(),
|
||
);
|
||
|
||
let (digest, words, _, _) = f.parse_digest(&test_opts());
|
||
|
||
assert_eq!(digest, "The Quick Brown. Fox Jumps Over. Ryan.".to_string());
|
||
assert_eq!(
|
||
words,
|
||
HashMap::from_iter([
|
||
(
|
||
"the".to_string(),
|
||
vec![FossickedWord {
|
||
position: 0,
|
||
weight: 1 * 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"quick".to_string(),
|
||
vec![FossickedWord {
|
||
position: 1,
|
||
weight: 2 * 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"brown".to_string(),
|
||
vec![FossickedWord {
|
||
position: 2,
|
||
weight: 2 * 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"fox".to_string(),
|
||
vec![FossickedWord {
|
||
position: 3,
|
||
weight: 1 * 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"jumps".to_string(),
|
||
vec![FossickedWord {
|
||
position: 4,
|
||
weight: 12,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"over".to_string(),
|
||
vec![FossickedWord {
|
||
position: 5,
|
||
weight: 12,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"ryan".to_string(),
|
||
vec![FossickedWord {
|
||
position: 6,
|
||
weight: 1,
|
||
original_word: None,
|
||
}]
|
||
)
|
||
])
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn parse_auto_weighted_file() {
|
||
let mut f = test_fossick(
|
||
[
|
||
"<html><body>",
|
||
"<h1>Pagefind</h1>",
|
||
"<h2>Pagefind</h2>",
|
||
"<h3>Pagefind</h3>",
|
||
"<h4>Pagefind</h4>",
|
||
"<h5>Pagefind</h5>",
|
||
"<h6>Pagefind</h6>",
|
||
"<p>Pagefind</p>",
|
||
"<div data-pagefind-weight='0'><h1>Pagefind</h1></div>",
|
||
"</body></html>",
|
||
]
|
||
.concat(),
|
||
);
|
||
|
||
let (_, words, _, _) = f.parse_digest(&test_opts());
|
||
|
||
assert_eq!(
|
||
words,
|
||
HashMap::from_iter([(
|
||
"pagefind".to_string(),
|
||
vec![
|
||
FossickedWord {
|
||
position: 0,
|
||
weight: 7 * 24,
|
||
original_word: None,
|
||
},
|
||
FossickedWord {
|
||
position: 1,
|
||
weight: 6 * 24,
|
||
original_word: None,
|
||
},
|
||
FossickedWord {
|
||
position: 2,
|
||
weight: 5 * 24,
|
||
original_word: None,
|
||
},
|
||
FossickedWord {
|
||
position: 3,
|
||
weight: 4 * 24,
|
||
original_word: None,
|
||
},
|
||
FossickedWord {
|
||
position: 4,
|
||
weight: 3 * 24,
|
||
original_word: None,
|
||
},
|
||
FossickedWord {
|
||
position: 5,
|
||
weight: 2 * 24,
|
||
original_word: None,
|
||
},
|
||
FossickedWord {
|
||
position: 6,
|
||
weight: 1 * 24,
|
||
original_word: None,
|
||
},
|
||
FossickedWord {
|
||
position: 7,
|
||
weight: 0 * 24,
|
||
original_word: None,
|
||
}
|
||
]
|
||
)])
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn parse_zero_weighted_compound_words() {
|
||
let mut f = test_fossick(
|
||
[
|
||
"<html><body>",
|
||
"<div data-pagefind-weight='0'>",
|
||
"<p>Simple text and compound.word.with.periods</p>",
|
||
"</div>",
|
||
"</body></html>",
|
||
]
|
||
.concat(),
|
||
);
|
||
|
||
let (_, words, _, _) = f.parse_digest(&test_opts());
|
||
|
||
for (_, word_positions) in words {
|
||
for position in word_positions {
|
||
assert_eq!(position.weight, 0, "Expected all words to have weight 0");
|
||
}
|
||
}
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn parse_bad_weights() {
|
||
let mut f = test_fossick(
|
||
[
|
||
"<html><body>",
|
||
"<p data-pagefind-weight='lots'>The</p>",
|
||
"<p data-pagefind-weight='99999999'>Quick</p>",
|
||
"<p data-pagefind-weight='-1234'>Brown</p>",
|
||
"<p data-pagefind-weight='65.4'>Fox</p>",
|
||
"</body></html>",
|
||
]
|
||
.concat(),
|
||
);
|
||
|
||
let (_, words, _, _) = f.parse_digest(&test_opts());
|
||
|
||
assert_eq!(
|
||
words,
|
||
HashMap::from_iter([
|
||
(
|
||
"the".to_string(),
|
||
vec![FossickedWord {
|
||
position: 0,
|
||
weight: 24,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"quick".to_string(),
|
||
vec![FossickedWord {
|
||
position: 1,
|
||
weight: 240,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"brown".to_string(),
|
||
vec![FossickedWord {
|
||
position: 2,
|
||
weight: 0,
|
||
original_word: None,
|
||
}]
|
||
),
|
||
(
|
||
"fox".to_string(),
|
||
vec![FossickedWord {
|
||
position: 3,
|
||
weight: 240,
|
||
original_word: None,
|
||
}]
|
||
)
|
||
])
|
||
);
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn parse_nbsp() {
|
||
let mut f = test_fossick(
|
||
[
|
||
"<html lang='ja'><body>",
|
||
"<p>Hello 👋</p>",
|
||
"</body></html>",
|
||
]
|
||
.concat(),
|
||
);
|
||
|
||
let (_, words, _, _) = f.parse_digest(&test_opts());
|
||
|
||
let mut words = words.keys().collect::<Vec<_>>();
|
||
words.sort();
|
||
assert_eq!(words, vec!["hello", "👋"]);
|
||
}
|
||
|
||
#[cfg(feature = "extended")]
|
||
#[tokio::test]
|
||
async fn parse_weights_through_segmentation() {
|
||
let mut f = test_fossick(
|
||
[
|
||
"<html lang='zh'><body>",
|
||
"<h1 id='my-title'>哎呀！ 我的错。</h1>",
|
||
"</body></html>",
|
||
]
|
||
.concat(),
|
||
);
|
||
|
||
let (content, words, _, _) = f.parse_digest(&test_opts());
|
||
|
||
let mut words = words.keys().collect::<Vec<_>>();
|
||
words.sort();
|
||
assert_eq!(words, vec!["哎呀", "我", "的", "错"]);
|
||
|
||
assert_eq!(
|
||
content,
|
||
"哎呀\u{200b}！ \u{200b}我\u{200b}的\u{200b}错\u{200b}。"
|
||
);
|
||
}
|
||
|
||
#[cfg(feature = "extended")]
|
||
#[tokio::test]
|
||
async fn segmentation_parity_when_presplitting() {
|
||
fn get_comparison_segmentations(full_input: &'static str) -> (Vec<String>, Vec<String>) {
|
||
let chunked_input = full_input
|
||
.split_whitespace()
|
||
.filter(|w| !w.starts_with("___"))
|
||
.collect::<Vec<_>>();
|
||
let clean_input = chunked_input.join(" ");
|
||
|
||
let mut legitimate_output = clean_input
|
||
.as_str()
|
||
.segment_str()
|
||
.filter(|w| w.chars().any(|c| !c.is_whitespace()))
|
||
.map(Into::into)
|
||
.collect::<Vec<_>>();
|
||
let mut chunked_output = chunked_input
|
||
.into_iter()
|
||
.flat_map(|inp| {
|
||
inp.segment_str()
|
||
.filter(|w| w.chars().any(|c| !c.is_whitespace()))
|
||
.collect::<Vec<_>>()
|
||
})
|
||
.map(Into::into)
|
||
.collect::<Vec<_>>();
|
||
|
||
legitimate_output.sort();
|
||
chunked_output.sort();
|
||
(legitimate_output, chunked_output)
|
||
}
|
||
{
|
||
let full_zh_input = "___PAGEFIND_AUTO_WEIGHT___7 擁有遠端帳號權限 ___END_PAGEFIND_WEIGHT___
|
||
|
||
我們建議大多數具有遠端帳號權限的使用者，採用 ___PAGEFIND_ANCHOR___a:0:my-link Certbot 這個 ACME 客戶端。它可以自動執行憑證的頒發、安裝，甚至不需要停止你的伺服器；Certbot 也提供專家模式，給不想要自動設定的使用者。Certbot 操作簡單，適用於許多系統；並且具有完善的文檔。參考 Certbot 官網，以獲取對於不同系統和網頁伺服器的操作說明。
|
||
|
||
如果 Certbot 不能滿足你的需求，或是你想嘗試別的客戶端，還有很多 ACME 用戶端可供選擇。在你選定 ACME 客戶端軟體後，請參閱該客戶端的文檔。
|
||
___PAGEFIND_WEIGHT___44
|
||
如果你正在嘗試使用不同的 ACME 用戶端，請使用我們的測試環境以免超過憑證頒發與更新的速率限制。
|
||
沒有遠端帳號權限
|
||
|
||
在沒有遠端帳號權限的情況下，最好的辦法是使用服務業者所提供的現有支援。如果你的業者支援 ___PAGEFIND_ANCHOR___a:1:my-second-link Let’s Encrypt，那麼他們就能幫助你申請免費憑證；安裝並設定自動更新。某些業者會需要你在控制介面或聯繫客服以開啟 Let’s Encrypt 服務。也有些業者會為所有客戶自動設定並安裝憑證。
|
||
|
||
查看支援 Let’s Encrypt 的業者列表，確認你提供商的是否有出現在列表上。如果有的話，請按照他們的文檔設定 Let’s Encrypt 憑證。 ___END_PAGEFIND_WEIGHT___";
|
||
|
||
let (legitimate_zh_output, chunked_zh_output) =
|
||
get_comparison_segmentations(full_zh_input);
|
||
assert_eq!(legitimate_zh_output, chunked_zh_output);
|
||
}
|
||
|
||
{
|
||
let full_zh_cn_input = "没有命令行访问权限
|
||
|
||
在没有命令行访问权限的情况下，___PAGEFIND_AUTO_WEIGHT___7 最好的办法是使用您托管服务提供商提供的内置功能。 支持 Let’s Encrypt 的服务商能替您自动完成免费证书的申请、安装、续期步骤。 某些服务商可能需要您在控制面板中开启相关选项， 也有一些服务商会自动为所有客户申请并安装证书。
|
||
|
||
如果您的服务商存在于我们的服务商列表中， 参照其文档设置 Let’s Encrypt ___END_PAGEFIND_WEIGHT___ 证书即可。
|
||
|
||
如果您的托管服务提供商不支持 ___PAGEFIND_ANCHOR___a:0:my-link Let’s Encrypt，您可以与他们联系请求支持。 我们尽力使添加 Let’s Encrypt 支持变得非常容易，提供商（注：非中国国内提供商）通常很乐意听取客户的建议！
|
||
|
||
如果您的托管服务提供商不想集成 Let’s Encrypt，但支持上传自定义证书，您可以在自己的计算机上安装 Certbot 并使用手动模式（Manual Mode）。 在手动模式下，您需要将指定文件上传到您的网站以证明您的控制权。 然后，Certbot 将获取您可以上传到提供商的证书。 我们不建议使用此选项，因为它非常耗时，并且您需要在证书过期时重复此步骤。 对于大多数人来说，最好从提供商处请求 Let’s Encrypt 支持。若您的提供商不打算兼容，建议您更换提供商。
|
||
获取帮助
|
||
|
||
如果您对选择 ACME 客户端，使用特定客户端或与 Let’s Encrypt 相关的任何其他内容有疑问，请前往我们的社区论坛获取帮助。";
|
||
|
||
let (legitimate_zh_cn_output, chunked_zh_cn_output) =
|
||
get_comparison_segmentations(full_zh_cn_input);
|
||
assert_eq!(legitimate_zh_cn_output, chunked_zh_cn_output);
|
||
}
|
||
|
||
{
|
||
let full_ja_input = "___PAGEFIND_AUTO_WEIGHT___7 シェルへのアクセス権を持っている場合
|
||
|
||
シェルアクセスができるほとんどの人には、Certbot という ACME クライアントを使うのがおすすめです。 ___END_PAGEFIND_WEIGHT___ 証明書の発行とインストールを、ダウンタイムゼロで自動化できます。 自動設定を使いたくない人のために、エキスパートモードも用意されています。 とても簡単に使え、多数のオペレーティングシステムで動作し、たくさんのドキュメントもあります。 Certbot のウェブサイトでは、各オペレーティングシステムやウェブサーバーごとの個別の設定方法について解説されています。
|
||
|
||
Certbot があなたの要件を満たさない場合や、他のクライアントを試してみたい場合には、Certbot の他にもたくさんの ACME クライアントが利用できます。 ACME クライアントを自分で選んだ場合は、そのクライアントのドキュメントを参照してください。
|
||
|
||
別の ACME クライアントを使って実験を行う場合は、 ___PAGEFIND_ANCHOR___a:0:my-link 私たちが用意したステージング環境を利用して、レート・リミットの制限を受けないように気をつけてください。
|
||
シェルへのアクセス権を持っていない場合
|
||
|
||
シェルアクセスができない場合に Let’s Encrypt を利用する一番良い方法は、ホスティング・プロバイダが用意したサポートを利用することです。 もし、あなたが利用するホスティング・プロバイダが Let’s Encrypt をサポートしている場合、あなたの代わりに無料の証明書をリクエスト、インストールし、自動的に最新の状態に更新してくれます。 一部のホスティング・プロバイダでは、この機能は自分で設定から有効にする必要がある場合があります。 それ以外のプロバイダでは、すべてのユーザーのために、自動で証明書が発行・インストールされるようになっています。
|
||
|
||
あなたが利用しているホスティング・プロバイダが Let’s Encrypt をサポートしているかどうかは、 ホスティング・プロバイダのリストで確認してください。 もしサポートされている場合は、ホスティング・プロバイダのドキュメンに書かれている Let’s Encrypt の設定方法に従ってください。";
|
||
|
||
let (legitimate_ja_output, chunked_ja_output) =
|
||
get_comparison_segmentations(full_ja_input);
|
||
assert_eq!(legitimate_ja_output, chunked_ja_output);
|
||
}
|
||
}
|
||
|
||
#[cfg(not(target_os = "windows"))]
|
||
#[test]
|
||
fn building_url() {
|
||
std::env::set_var("PAGEFIND_SITE", "hello/world");
|
||
let config =
|
||
PagefindInboundConfig::with_layers(&[Layer::Env(Some("PAGEFIND_".into()))]).unwrap();
|
||
let opts = SearchOptions::load(config).unwrap();
|
||
|
||
let cwd = std::env::current_dir().unwrap();
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/index.html".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/");
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/about/index.html".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/about/");
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/about.html".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/about.html");
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/about/index.htm".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/about/index.htm");
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/index.html".into());
|
||
let root: PathBuf = cwd.join::<PathBuf>("hello".into());
|
||
assert_eq!(&build_url(&p, Some(&root), &opts), "/world/");
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/100%/C#/why?.html".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/100%25/C%23/why%3F.html");
|
||
}
|
||
|
||
#[cfg(not(target_os = "windows"))]
|
||
#[test]
|
||
fn building_urls_with_characters_that_need_encoding() {
|
||
std::env::set_var("PAGEFIND_SITE", "hello/world");
|
||
let config =
|
||
PagefindInboundConfig::with_layers(&[Layer::Env(Some("PAGEFIND_".into()))]).unwrap();
|
||
let opts = SearchOptions::load(config).unwrap();
|
||
|
||
let cwd = std::env::current_dir().unwrap();
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/Müsli.html".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/M%C3%BCsli.html");
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/Müsli/index.html".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/M%C3%BCsli/");
|
||
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/a b/c#d?e.html".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/a%20b/c%23d%3Fe.html");
|
||
|
||
// A literal percent in the file name is encoded, so the URL points back at
|
||
// the file that is on disk rather than at the one it looks like it names.
|
||
let p: PathBuf = cwd.join::<PathBuf>("hello/world/M%C3%BCsli.html".into());
|
||
assert_eq!(&build_url(&p, None, &opts), "/M%25C3%25BCsli.html");
|
||
}
|
||
|
||
#[cfg(target_os = "windows")]
|
||
#[test]
|
||
fn building_windows_urls() {
|
||
std::env::set_var("PAGEFIND_SITE", "C:\\hello\\world");
|
||
let config =
|
||
PagefindInboundConfig::with_layers(&[Layer::Env(Some("PAGEFIND_".into()))]).unwrap();
|
||
let opts = SearchOptions::load(config).unwrap();
|
||
|
||
let p: PathBuf = "C:\\hello\\world\\index.html".into();
|
||
assert_eq!(&build_url(&p, None, &opts), "/");
|
||
|
||
let p: PathBuf = "C:\\hello\\world\\about\\index.html".into();
|
||
assert_eq!(&build_url(&p, None, &opts), "/about/");
|
||
|
||
let p: PathBuf = "C:\\hello\\world\\about\\index.htm".into();
|
||
assert_eq!(&build_url(&p, None, &opts), "/about/index.htm");
|
||
}
|
||
}
|