Upstream Pagefind/pagefind state of the touched files before f1eedafd11
This commit is contained in:
upstream committed
1 parent
b572112ae7
commit
1308f3c1bb
10 files changed
+1706
-79
No files matched your search
@@ -0,0 +1,303 @@
|
||||
import { PagefindService } from "./service.js";
|
||||
import { decode } from "./encoding.js";
|
||||
|
||||
/**
|
||||
* @typedef {import('pagefindInternal').InternalResponseCallback} InternalResponseCallback
|
||||
* @typedef {import('pagefindInternal').InternalResponsePayload} InternalResponsePayload
|
||||
*/
|
||||
|
||||
/**
|
||||
* @type {PagefindService?}
|
||||
*/
|
||||
let persistentService;
|
||||
const launch = () => {
|
||||
if (!persistentService) {
|
||||
persistentService = new PagefindService();
|
||||
}
|
||||
return persistentService;
|
||||
}
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @param {function(any): void} resolve
|
||||
* @param {function(any): void} reject
|
||||
* @param {InternalResponseCallback} response_callback
|
||||
* @param {function(InternalResponsePayload): T} resultFn
|
||||
*/
|
||||
const handleApiResponse = (resolve, reject, { exception, err, result }, resultFn) => {
|
||||
if (exception) {
|
||||
reject(exception);
|
||||
} else {
|
||||
resolve({
|
||||
errors: err ? [err.message] : [],
|
||||
...(result ? resultFn(result) : {})
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {import('pagefindService').NewIndexResponse} NewIndexResponse
|
||||
*
|
||||
* @param {import('pagefindService').PagefindServiceConfig=} config
|
||||
* @type {import('pagefindService').createIndex}
|
||||
* */
|
||||
export const createIndex = (config) => new Promise((resolve, reject) => {
|
||||
|
||||
// TODO: Validate `config` here, instead of waiting for the backend to throw an error.
|
||||
// Ideally we create a global Pagefind config JSON schema that (a subset of) can be used here.
|
||||
|
||||
const action = 'NewIndex';
|
||||
launch().sendMessage(
|
||||
{
|
||||
type: action,
|
||||
config: {
|
||||
root_selector: config?.rootSelector,
|
||||
exclude_selectors: config?.excludeSelectors,
|
||||
force_language: config?.forceLanguage,
|
||||
verbose: config?.verbose,
|
||||
logfile: config?.logfile,
|
||||
keep_index_url: config?.keepIndexUrl,
|
||||
write_playground: config?.writePlayground,
|
||||
include_characters: config?.includeCharacters,
|
||||
}
|
||||
}, (response) => {
|
||||
/** @type {function(InternalResponsePayload): Omit<NewIndexResponse, 'errors'>?} */
|
||||
const successCallback = (success) => {
|
||||
if (success.type !== action) {
|
||||
reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
index: indexFns(success.index_id),
|
||||
}
|
||||
};
|
||||
handleApiResponse(resolve, reject, response, successCallback);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* @type {import('pagefindService').close}
|
||||
*/
|
||||
export const close = () => new Promise((resolve, reject) => {
|
||||
persistentService?.close(null);
|
||||
persistentService = null;
|
||||
resolve(null);
|
||||
});
|
||||
|
||||
/**
|
||||
* @param {number} indexId
|
||||
* @returns {import ('pagefindService').PagefindIndex}
|
||||
*/
|
||||
const indexFns = (indexId) => {
|
||||
return {
|
||||
addHTMLFile: (file) => addHTMLFile(indexId, file),
|
||||
addCustomRecord: (record) => addCustomRecord(indexId, record),
|
||||
addDirectory: (dir) => addDirectory(indexId, dir),
|
||||
writeFiles: (options) => writeFiles(indexId, options),
|
||||
getFiles: () => getFiles(indexId),
|
||||
deleteIndex: () => deleteIndex(indexId)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {import('pagefindService').NewFileResponse} NewFileResponse
|
||||
*
|
||||
* @param {number} indexId
|
||||
* @param {import('pagefindService').HTMLFile} file
|
||||
* @returns {Promise<NewFileResponse>}
|
||||
*/
|
||||
const addHTMLFile = (indexId, file) => new Promise((resolve, reject) => {
|
||||
const action = 'AddFile';
|
||||
const responseAction = 'IndexedFile';
|
||||
launch().sendMessage(
|
||||
{
|
||||
type: action,
|
||||
index_id: indexId,
|
||||
file_path: file.sourcePath,
|
||||
url: file.url,
|
||||
file_contents: file.content
|
||||
}, (response) => {
|
||||
/** @type {function(InternalResponsePayload): Omit<NewFileResponse, 'errors'>?} */
|
||||
const successCallback = (success) => {
|
||||
if (success.type !== responseAction) {
|
||||
reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
file: {
|
||||
uniqueWords: success.page_word_count,
|
||||
url: success.page_url,
|
||||
meta: success.page_meta,
|
||||
}
|
||||
}
|
||||
};
|
||||
handleApiResponse(resolve, reject, response, successCallback);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* @param {number} indexId
|
||||
* @param {import('pagefindService').CustomRecord} record
|
||||
* @returns {Promise<NewFileResponse>}
|
||||
*/
|
||||
const addCustomRecord = (indexId, record) => new Promise((resolve, reject) => {
|
||||
const action = 'AddRecord';
|
||||
const responseAction = 'IndexedFile';
|
||||
launch().sendMessage(
|
||||
{
|
||||
type: action,
|
||||
index_id: indexId,
|
||||
url: record.url,
|
||||
content: record.content,
|
||||
language: record.language,
|
||||
meta: record.meta,
|
||||
filters: record.filters,
|
||||
sort: record.sort,
|
||||
}, (response) => {
|
||||
/** @type {function(InternalResponsePayload): Omit<NewFileResponse, 'errors'>?} */
|
||||
const successCallback = (success) => {
|
||||
if (success.type !== responseAction) {
|
||||
reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
file: {
|
||||
uniqueWords: success.page_word_count,
|
||||
url: success.page_url,
|
||||
meta: success.page_meta,
|
||||
}
|
||||
}
|
||||
};
|
||||
handleApiResponse(resolve, reject, response, successCallback);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
/**
|
||||
* @typedef {import('pagefindService').IndexingResponse} IndexingResponse
|
||||
*
|
||||
* @param {number} indexId
|
||||
* @param {import('pagefindService').SiteDirectory} dir
|
||||
* @returns {Promise<IndexingResponse>}
|
||||
*/
|
||||
const addDirectory = (indexId, dir) => new Promise((resolve, reject) => {
|
||||
const action = 'AddDir';
|
||||
const responseAction = 'IndexedDir';
|
||||
launch().sendMessage(
|
||||
{
|
||||
type: action,
|
||||
index_id: indexId,
|
||||
path: dir.path,
|
||||
glob: dir.glob
|
||||
}, (response) => {
|
||||
/** @type {function(InternalResponsePayload): Omit<IndexingResponse, 'errors'>?} */
|
||||
const successCallback = (success) => {
|
||||
if (success.type !== responseAction) {
|
||||
reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
page_count: success.page_count
|
||||
}
|
||||
};
|
||||
handleApiResponse(resolve, reject, response, successCallback);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* @typedef {import ('pagefindService').WriteFilesResponse} WriteFilesResponse
|
||||
*
|
||||
* @param {number} indexId
|
||||
* @param {import('pagefindService').WriteOptions=} options
|
||||
* @returns {Promise<WriteFilesResponse>}
|
||||
*/
|
||||
const writeFiles = (indexId, options) => new Promise((resolve, reject) => {
|
||||
const action = 'WriteFiles';
|
||||
launch().sendMessage(
|
||||
{
|
||||
type: action,
|
||||
index_id: indexId,
|
||||
output_path: options?.outputPath
|
||||
}, (response) => {
|
||||
/** @type {function(InternalResponsePayload): Omit<WriteFilesResponse, 'errors'>?} */
|
||||
const successCallback = (success) => {
|
||||
if (success.type !== action) {
|
||||
reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
outputPath: success.output_path
|
||||
}
|
||||
};
|
||||
handleApiResponse(resolve, reject, response, successCallback);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* @typedef {import ('pagefindService').GetFilesResponse} GetFilesResponse
|
||||
*
|
||||
* @param {number} indexId
|
||||
* @returns {Promise<GetFilesResponse>}
|
||||
*/
|
||||
const getFiles = (indexId) => new Promise((resolve, reject) => {
|
||||
const action = 'GetFiles';
|
||||
launch().sendMessage(
|
||||
{
|
||||
type: action,
|
||||
index_id: indexId,
|
||||
}, (response) => {
|
||||
/** @type {function(InternalResponsePayload): Omit<GetFilesResponse, 'errors'>?} */
|
||||
const successCallback = (success) => {
|
||||
if (success.type !== action) {
|
||||
reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
files: success.files.map(file => {
|
||||
return {
|
||||
path: file.path,
|
||||
content: decode(file.content)
|
||||
}
|
||||
})
|
||||
}
|
||||
};
|
||||
handleApiResponse(resolve, reject, response, successCallback);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* @param {number} indexId
|
||||
* @returns {Promise<null>}
|
||||
*/
|
||||
const deleteIndex = (indexId) => new Promise((resolve, reject) => {
|
||||
const action = 'DeleteIndex';
|
||||
launch().sendMessage(
|
||||
{
|
||||
type: action,
|
||||
index_id: indexId,
|
||||
}, (response) => {
|
||||
/** @type {function(InternalResponsePayload): Omit<GetFilesResponse, 'errors'>?} */
|
||||
const successCallback = (success) => {
|
||||
if (success.type !== action) {
|
||||
reject(`Message returned from backend should have been ${action}, but was ${success.type}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
handleApiResponse(resolve, reject, response, successCallback);
|
||||
}
|
||||
);
|
||||
});
|
||||
Vendored
+227
@@ -0,0 +1,227 @@
|
||||
/**
|
||||
* Create a new Pagefind index that files can be added to
|
||||
*/
|
||||
export function createIndex(config?: PagefindServiceConfig): Promise<NewIndexResponse>;
|
||||
|
||||
/**
|
||||
* Close the Pagefind service and clean up, stopping the binary altogether.
|
||||
*
|
||||
* Service _will_ be restarted if any new calls are made to any other function.
|
||||
*
|
||||
* Calling functions on an existing index after calling close() is undefined behavior,
|
||||
* as that index may map to a new index after the service has restarted.
|
||||
*/
|
||||
export function close(): Promise<null>;
|
||||
|
||||
export interface PagefindServiceConfig {
|
||||
/**
|
||||
* The element Pagefind should treat as the root of the document, defaults to `html`.
|
||||
* Usually you will want to use the data-pagefind-body attribute instead.
|
||||
* @example ".my-html-outer"
|
||||
*/
|
||||
rootSelector?: string,
|
||||
/**
|
||||
* Custom selectors that Pagefind should ignore when indexing.
|
||||
* Usually you will want to use the data-pagefind-ignore attribute instead.
|
||||
* @example ["svg", ".my-code-blocks"]
|
||||
*/
|
||||
excludeSelectors?: string[],
|
||||
/**
|
||||
* Ignore any detected languages and index the whole site as a single language.
|
||||
* Expects an ISO 639-1 code.
|
||||
*/
|
||||
forceLanguage?: string,
|
||||
/**
|
||||
* Print verbose logging while indexing the site. Does not impact the web-facing search.
|
||||
* When running as a service, only impacts the logfile (if present).
|
||||
*/
|
||||
verbose?: boolean,
|
||||
/**
|
||||
* Path to a logfile to write to. Will replace the file on each run.
|
||||
*/
|
||||
logfile?: string,
|
||||
/**
|
||||
* Keep `index.html` at the end of search result paths.
|
||||
* Defaults to false, stripping `index.html`.
|
||||
*/
|
||||
keepIndexUrl?: boolean,
|
||||
/**
|
||||
* When writing or outputting files, also write the Pagefind playground to /pagefind/playground/.
|
||||
* Defaults to false, ensuring the playground isn't available on a live site.
|
||||
*/
|
||||
writePlayground?: boolean,
|
||||
/**
|
||||
* Include these characters when indexing and searching words.
|
||||
* Useful for sites documenting technical topics such as programming languages.
|
||||
* @example "<>$"
|
||||
*/
|
||||
includeCharacters?: string,
|
||||
}
|
||||
|
||||
|
||||
export interface NewIndexResponse {
|
||||
errors: string[],
|
||||
index?: PagefindIndex
|
||||
}
|
||||
|
||||
/**
|
||||
* A Pagefind index that exists in the backend service before being built
|
||||
*/
|
||||
export interface PagefindIndex {
|
||||
addHTMLFile: typeof addHTMLFile,
|
||||
addCustomRecord: typeof addCustomRecord,
|
||||
addDirectory: typeof addDirectory,
|
||||
writeFiles: typeof writeFiles,
|
||||
getFiles: typeof getFiles,
|
||||
deleteIndex: typeof deleteIndex,
|
||||
}
|
||||
|
||||
/**
|
||||
* Index an HTML file that isn't on disk
|
||||
*/
|
||||
declare function addHTMLFile(file: HTMLFile): Promise<NewFileResponse>;
|
||||
/**
|
||||
* Index a custom record that isn't backed by an HTML file
|
||||
*/
|
||||
declare function addCustomRecord(record: CustomRecord): Promise<NewFileResponse>;
|
||||
/**
|
||||
* Index a directory of HTML files from disk
|
||||
*/
|
||||
declare function addDirectory(path: SiteDirectory): Promise<IndexingResponse>;
|
||||
|
||||
/**
|
||||
* The data required for Pagefind to index an HTML file that isn't on disk
|
||||
* @example
|
||||
* {
|
||||
* sourcePath: "about/index.html",
|
||||
* content: "<html lang='en'><body><h1>Meet the team</h1></body></html>"
|
||||
* }
|
||||
*/
|
||||
export interface HTMLFile {
|
||||
/**
|
||||
* The source path of the HTML file if it were to exist on disk.
|
||||
* Must be a relative path, or an absolute path within the current working directory.
|
||||
* Pagefind will compute the result URL from this path.
|
||||
*
|
||||
* If not supplied, url must be supplied.
|
||||
*
|
||||
* @example "about/index.html"
|
||||
* @example "/Users/user/Documents/site/about/index.html"
|
||||
*/
|
||||
sourcePath?: string,
|
||||
/**
|
||||
* An explicit URL to use, instead of having Pagefind
|
||||
* compute the URL based on the sourcePath.
|
||||
*
|
||||
* If not supplied, sourcePath must be supplied.
|
||||
*
|
||||
* @example "/about/"
|
||||
*/
|
||||
url?: string,
|
||||
/** The source HTML content of the file to be parsed */
|
||||
content: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The data required for Pagefind to index a custom record that isn't backed by an HTML file
|
||||
* @example
|
||||
* {
|
||||
* url: "/about/",
|
||||
* content: "Meet the team",
|
||||
* language: "en"
|
||||
* }
|
||||
*/
|
||||
export interface CustomRecord {
|
||||
/** The output URL of this record. Pagefind will not alter this */
|
||||
url: string,
|
||||
/** The raw content of this record */
|
||||
content: string,
|
||||
/** What language is this record written in. Multiple languages will be split into separate indexes. Expects an ISO 639-1 code. */
|
||||
language: string,
|
||||
/** The metadata to attach to this record. Supplying a `title` is highly recommended */
|
||||
meta?: Record<string, string>,
|
||||
/** The filters to attach to this record */
|
||||
filters?: Record<string, string[]>,
|
||||
/** The sort keys to attach to this record */
|
||||
sort?: Record<string, string>
|
||||
}
|
||||
|
||||
/**
|
||||
* The data required for Pagefind to index the files in a directory
|
||||
* @example
|
||||
* {
|
||||
* path: "public",
|
||||
* glob: "**\/*.{html}"
|
||||
* }
|
||||
*/
|
||||
export interface SiteDirectory {
|
||||
/**
|
||||
* The path to the directory to index.
|
||||
* If relative, is relative to the cwd.
|
||||
*/
|
||||
path: string,
|
||||
/** Optionally, a custom glob to evaluate for finding files. Default to all HTML files. */
|
||||
glob?: string
|
||||
}
|
||||
|
||||
export interface IndexingResponse {
|
||||
errors: string[],
|
||||
page_count: number
|
||||
}
|
||||
|
||||
export interface NewFileResponse {
|
||||
errors: string[],
|
||||
file: NewFile
|
||||
}
|
||||
|
||||
/**
|
||||
* Details about a new file that has been successfully added to the Pagefind index
|
||||
*/
|
||||
export interface NewFile {
|
||||
uniqueWords: number,
|
||||
url: string,
|
||||
meta: Record<string, string>
|
||||
}
|
||||
|
||||
/**
|
||||
* Write the index files to disk
|
||||
*/
|
||||
declare function writeFiles(options?: WriteOptions): Promise<WriteFilesResponse>;
|
||||
|
||||
/**
|
||||
* Options for writing a Pagefind index to disk
|
||||
*/
|
||||
export interface WriteOptions {
|
||||
/**
|
||||
* The path of the pagefind bundle directory to write to disk.
|
||||
* If relative, is relative to the cwd.
|
||||
* @example "./public/pagefind"
|
||||
*/
|
||||
outputPath: string
|
||||
}
|
||||
|
||||
|
||||
export interface WriteFilesResponse {
|
||||
errors: string[],
|
||||
outputPath: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an in-memory copy of the built index files
|
||||
*/
|
||||
declare function getFiles(): Promise<GetFilesResponse>;
|
||||
|
||||
export interface GetFilesResponse {
|
||||
errors: string[],
|
||||
files: IndexFile[]
|
||||
}
|
||||
|
||||
export interface IndexFile {
|
||||
path: string,
|
||||
content: Uint8Array
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete this index and clear it from memory
|
||||
*/
|
||||
declare function deleteIndex(): Promise<null>;
|
||||
Vendored
+154
@@ -0,0 +1,154 @@
|
||||
// Requests to the backend.
|
||||
|
||||
/**
|
||||
* The raw object passed to the backend binary
|
||||
*/
|
||||
export interface InternalServiceRequest {
|
||||
message_id?: number,
|
||||
payload: InternalRequestPayload
|
||||
}
|
||||
|
||||
/**
|
||||
* The payload describing an action to the backend binary
|
||||
*/
|
||||
export type InternalRequestPayload =
|
||||
| InternalNewIndexRequest
|
||||
| InternalAddFileRequest
|
||||
| InternalAddRecordRequest
|
||||
| InternalAddDirRequest
|
||||
| InternalWriteFilesRequest
|
||||
| InternalGetFilesRequest
|
||||
| InternalDeleteIndexRequest;
|
||||
|
||||
export interface InternalNewIndexRequest {
|
||||
type: 'NewIndex',
|
||||
config?: InternalPagefindServiceConfig
|
||||
}
|
||||
|
||||
export interface InternalPagefindServiceConfig {
|
||||
root_selector?: string,
|
||||
exclude_selectors?: string[],
|
||||
force_language?: string,
|
||||
verbose?: boolean,
|
||||
logfile?: string,
|
||||
keep_index_url?: boolean,
|
||||
write_playground?: boolean,
|
||||
include_characters?: string,
|
||||
}
|
||||
|
||||
export interface InternalAddFileRequest {
|
||||
type: 'AddFile',
|
||||
index_id: number,
|
||||
file_path?: string,
|
||||
url?: string,
|
||||
file_contents: string
|
||||
}
|
||||
|
||||
export interface InternalAddRecordRequest {
|
||||
type: 'AddRecord',
|
||||
index_id: number,
|
||||
url: string,
|
||||
content: string,
|
||||
language: string,
|
||||
meta?: Record<string, string>,
|
||||
filters?: Record<string, string[]>,
|
||||
sort?: Record<string, string>
|
||||
}
|
||||
|
||||
export interface InternalAddDirRequest {
|
||||
type: 'AddDir',
|
||||
index_id: number,
|
||||
path: string,
|
||||
glob?: string
|
||||
}
|
||||
|
||||
export interface InternalWriteFilesRequest {
|
||||
type: 'WriteFiles',
|
||||
index_id: number,
|
||||
output_path?: string
|
||||
}
|
||||
|
||||
export interface InternalGetFilesRequest {
|
||||
type: 'GetFiles',
|
||||
index_id: number
|
||||
}
|
||||
|
||||
export interface InternalDeleteIndexRequest {
|
||||
type: 'DeleteIndex',
|
||||
index_id: number
|
||||
}
|
||||
|
||||
// Responses from the backend.
|
||||
|
||||
/**
|
||||
* The raw object returned from the backend binary
|
||||
*/
|
||||
export interface InternalServiceResponse {
|
||||
message_id: number,
|
||||
payload: InternalResponsePayload | InternalResponseError
|
||||
}
|
||||
|
||||
/**
|
||||
* The response payload in the case of an error
|
||||
*/
|
||||
export interface InternalResponseError {
|
||||
type: 'Error',
|
||||
original_message?: string,
|
||||
message: string
|
||||
}
|
||||
|
||||
/**
|
||||
* The response payload in the case of a success
|
||||
*/
|
||||
export type InternalResponsePayload =
|
||||
| InternalNewIndexResponse
|
||||
| InternalIndexedFileResponse
|
||||
| InternalIndexedDirResponse
|
||||
| InternalWriteFilesResponse
|
||||
| InternalGetFilesResponse
|
||||
| InternalDeleteIndexResponse;
|
||||
|
||||
export interface InternalNewIndexResponse {
|
||||
type: 'NewIndex',
|
||||
index_id: number
|
||||
}
|
||||
|
||||
export interface InternalIndexedFileResponse {
|
||||
type: 'IndexedFile',
|
||||
page_word_count: number,
|
||||
page_url: string,
|
||||
page_meta: Record<string, string>
|
||||
}
|
||||
|
||||
export interface InternalIndexedDirResponse {
|
||||
type: 'IndexedDir',
|
||||
page_count: number
|
||||
}
|
||||
|
||||
export interface InternalWriteFilesResponse {
|
||||
type: 'WriteFiles',
|
||||
output_path: string,
|
||||
}
|
||||
|
||||
export interface InternalGetFilesResponse {
|
||||
type: 'GetFiles',
|
||||
files: InternalSyntheticFile[],
|
||||
}
|
||||
|
||||
export interface InternalSyntheticFile {
|
||||
path: string,
|
||||
content: string
|
||||
}
|
||||
|
||||
export interface InternalDeleteIndexResponse {
|
||||
type: 'DeleteIndex'
|
||||
}
|
||||
|
||||
/**
|
||||
* What the service returns to the wrapping javascript detailing a response
|
||||
*/
|
||||
export interface InternalResponseCallback {
|
||||
exception: Error | null,
|
||||
err: InternalResponseError | null,
|
||||
result: InternalResponsePayload | null
|
||||
}
|
||||
@@ -0,0 +1,284 @@
|
||||
import logging
|
||||
import base64
|
||||
from typing import Any, Dict, List, Optional, Sequence, TypedDict, cast
|
||||
|
||||
from ..service import PagefindService
|
||||
from ..service.types import (
|
||||
InternalAddDirRequest,
|
||||
InternalAddFileRequest,
|
||||
InternalAddRecordRequest,
|
||||
InternalDeleteIndexRequest,
|
||||
InternalGetFilesRequest,
|
||||
InternalGetFilesResponse,
|
||||
InternalIndexedDirResponse,
|
||||
InternalIndexedFileResponse,
|
||||
InternalDecodedFile,
|
||||
InternalWriteFilesRequest,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class IndexConfig(TypedDict, total=False):
|
||||
root_selector: Optional[str]
|
||||
"""
|
||||
The root selector to use for the index.
|
||||
If not supplied, Pagefind will use the ``<html>`` tag.
|
||||
"""
|
||||
exclude_selectors: Optional[Sequence[str]]
|
||||
"""Extra element selectors that Pagefind should ignore when indexing."""
|
||||
force_language: Optional[str]
|
||||
"""
|
||||
Ignores any detected languages and creates a single index for the entire site as the
|
||||
provided language. Expects an ISO 639-1 code, such as ``en`` or ``pt``.
|
||||
"""
|
||||
verbose: Optional[bool]
|
||||
"""
|
||||
Prints extra logging while indexing the site. Only affects the CLI, does not impact
|
||||
web-facing search.
|
||||
"""
|
||||
logfile: Optional[str]
|
||||
"""
|
||||
A path to a file to log indexing output to in addition to stdout.
|
||||
The file will be created if it doesn't exist and overwritten on each run.
|
||||
"""
|
||||
keep_index_url: Optional[bool]
|
||||
"""Whether to keep ``index.html`` at the end of search result paths.
|
||||
|
||||
By default, a file at ``animals/cat/index.html`` will be given the URL
|
||||
``/animals/cat/``. Setting this option to ``true`` will result in the URL
|
||||
``/animals/cat/index.html``.
|
||||
"""
|
||||
write_playground: Optional[bool]
|
||||
"""When writing or outputting files, also write the Pagefind playground to /pagefind/playground/.
|
||||
|
||||
Defaults to false, ensuring the playground isn't available on a live site.
|
||||
"""
|
||||
include_characters: Optional[str]
|
||||
"""Include these characters when indexing and searching words.
|
||||
|
||||
Useful for sites documenting technical topics such as programming languages.
|
||||
"""
|
||||
output_path: Optional[str]
|
||||
"""
|
||||
The folder to output the search bundle into, relative to the processed site.
|
||||
Defaults to ``pagefind``.
|
||||
"""
|
||||
|
||||
|
||||
class PagefindIndex:
|
||||
"""Manages a Pagefind index.
|
||||
|
||||
``PagefindIndex`` operates as an async contextmanager.
|
||||
Entering the context starts a backing Pagefind service and creates an in-memory index in the backing service.
|
||||
Exiting the context writes the in-memory index to disk and then shuts down the backing Pagefind service.
|
||||
|
||||
Each method of ``PagefindIndex`` that talks to the backing Pagefind service can raise errors.
|
||||
If an exception is is rased inside ``PagefindIndex``'s context, the context closes without writing the index files to disk.
|
||||
|
||||
``PagefindIndex`` optionally takes a configuration dictionary that can apply parts of the [Pagefind CLI config](/docs/config-options/). The options available at this level are:
|
||||
|
||||
See the relevant documentation for these configuration options in the
|
||||
`Configuring the Pagefind CLI <https://pagefind.app/docs/config-options/>` documentation.
|
||||
"""
|
||||
|
||||
_service: Optional["PagefindService"] = None
|
||||
_index_id: Optional[int] = None
|
||||
_config: Optional[IndexConfig] = None
|
||||
"""Note that config should be immutable."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Optional[IndexConfig] = None,
|
||||
*,
|
||||
_service: Optional["PagefindService"] = None,
|
||||
_index_id: Optional[int] = None,
|
||||
):
|
||||
self._service = _service
|
||||
self._index_id = _index_id
|
||||
self._config = config
|
||||
|
||||
async def _start(self) -> "PagefindIndex":
|
||||
"""Start the backing Pagefind service and create an in-memory index."""
|
||||
assert self._index_id is None
|
||||
assert self._service is None
|
||||
self._service = await PagefindService().launch()
|
||||
_index = await self._service.create_index(self._config)
|
||||
self._index_id = _index._index_id
|
||||
return self
|
||||
|
||||
async def add_html_file(
|
||||
self,
|
||||
*,
|
||||
content: str,
|
||||
source_path: Optional[str] = None,
|
||||
url: Optional[str] = None,
|
||||
) -> InternalIndexedFileResponse:
|
||||
"""Add an HTML file to the index.
|
||||
|
||||
:param content: The source HTML content of the file to be parsed.
|
||||
:param source_path: The source path of the HTML file would have on disk. \
|
||||
Must be a relative path, or an absolute path within the current working directory. \
|
||||
Pagefind will compute the result URL from this path.
|
||||
:param url: an explicit URL to use, instead of having Pagefind compute the \
|
||||
URL based on the source_path. If not supplied, source_path must be supplied.
|
||||
"""
|
||||
assert self._service is not None
|
||||
assert self._index_id is not None
|
||||
result = await self._service.send(
|
||||
InternalAddFileRequest(
|
||||
type="AddFile",
|
||||
index_id=self._index_id,
|
||||
url=url,
|
||||
file_contents=content,
|
||||
file_path=source_path,
|
||||
)
|
||||
)
|
||||
assert result["type"] == "IndexedFile"
|
||||
return cast(InternalIndexedFileResponse, result)
|
||||
|
||||
async def add_directory(
|
||||
self, path: str, *, glob: Optional[str] = None
|
||||
) -> InternalIndexedDirResponse:
|
||||
"""Indexes a directory from disk using the standard Pagefind indexing behaviour.
|
||||
|
||||
This is equivalent to running the Pagefind binary with ``--site <dir>``.
|
||||
|
||||
:param path: the path to the directory to index. If the `path` provided is relative, \
|
||||
it will be relative to the current working directory of your Python process.
|
||||
:param glob: a glob pattern to filter files in the directory. If not provided, all \
|
||||
files matching ``**.{html}`` are indexed. For more information on glob patterns, \
|
||||
see the `Wax patterns documentation <https://github.com/olson-sean-k/wax#patterns>`.
|
||||
"""
|
||||
assert self._service is not None
|
||||
assert self._index_id is not None
|
||||
result = await self._service.send(
|
||||
InternalAddDirRequest(
|
||||
type="AddDir",
|
||||
index_id=self._index_id,
|
||||
path=path,
|
||||
glob=glob,
|
||||
)
|
||||
)
|
||||
assert result["type"] == "IndexedDir"
|
||||
return cast(InternalIndexedDirResponse, result)
|
||||
|
||||
async def get_files(self) -> List[InternalDecodedFile]:
|
||||
"""Get raw data of all files in the Pagefind index.
|
||||
|
||||
WATCH OUT: this method emits all files. This can be a lot of data, and
|
||||
this amount of data can cause reading from the subprocess pipes to deadlock.
|
||||
|
||||
STRICTLY PREFER calling ``self.write_files()``.
|
||||
"""
|
||||
assert self._service is not None
|
||||
assert self._index_id is not None
|
||||
|
||||
response = await self._service.send(
|
||||
InternalGetFilesRequest(type="GetFiles", index_id=self._index_id)
|
||||
)
|
||||
assert response["type"] == "GetFiles"
|
||||
files = cast(InternalGetFilesResponse, response)["files"]
|
||||
|
||||
decoded_files = [
|
||||
{"path": file["path"], "content": base64.b64decode(file["content"])}
|
||||
for file in files
|
||||
]
|
||||
|
||||
return cast(List[InternalDecodedFile], decoded_files)
|
||||
|
||||
async def delete_index(self) -> None:
|
||||
"""
|
||||
Deletes the data for the given index from its backing Pagefind service.
|
||||
Doesn't affect any written files or data returned by ``get_files()``.
|
||||
"""
|
||||
assert self._service is not None
|
||||
assert self._index_id is not None
|
||||
result = await self._service.send(
|
||||
InternalDeleteIndexRequest(type="DeleteIndex", index_id=self._index_id)
|
||||
)
|
||||
assert result["type"] == "DeleteIndex"
|
||||
self._index_id = None
|
||||
self._service = None
|
||||
|
||||
async def add_custom_record(
|
||||
self,
|
||||
*,
|
||||
url: str,
|
||||
content: str,
|
||||
language: str,
|
||||
meta: Optional[Dict[str, str]] = None,
|
||||
filters: Optional[Dict[str, List[str]]] = None,
|
||||
sort: Optional[Dict[str, str]] = None,
|
||||
) -> InternalIndexedFileResponse:
|
||||
"""Add a direct record to the Pagefind index.
|
||||
|
||||
This method is useful for adding non-HTML content to the search results.
|
||||
|
||||
:param content: the raw content of this record.
|
||||
:param url: the output URL of this record. Pagefind will not alter this.
|
||||
:param language: ISO 639-1 code of the language this record is written in.
|
||||
:param meta: the metadata to attach to this record. Supplying a ``title`` is highly recommended.
|
||||
:param filters: the filters to attach to this record. Filters are used to group records together.
|
||||
:param sort: the sort keys to attach to this record.
|
||||
"""
|
||||
assert self._service is not None
|
||||
assert self._index_id is not None
|
||||
result = await self._service.send(
|
||||
InternalAddRecordRequest(
|
||||
type="AddRecord",
|
||||
index_id=self._index_id,
|
||||
url=url,
|
||||
content=content,
|
||||
language=language,
|
||||
meta=meta,
|
||||
filters=filters,
|
||||
sort=sort,
|
||||
)
|
||||
)
|
||||
assert result["type"] == "IndexedFile"
|
||||
return cast(InternalIndexedFileResponse, result)
|
||||
|
||||
async def write_files(self, output_path: Optional[str] = None) -> None:
|
||||
"""Write the index files to disk.
|
||||
|
||||
If you're using PagefindIndex as a context manager, there's no need to call this method:
|
||||
if no error occurred, closing the context automatically writes the index files to disk.
|
||||
|
||||
:param output_path: a path to override the configured output path for the index.
|
||||
"""
|
||||
assert self._service is not None
|
||||
assert self._index_id is not None
|
||||
if not output_path:
|
||||
if not self._config:
|
||||
output_path = None
|
||||
else:
|
||||
output_path = self._config.get("output_path")
|
||||
|
||||
result = await self._service.send(
|
||||
InternalWriteFilesRequest(
|
||||
type="WriteFiles",
|
||||
index_id=self._index_id,
|
||||
output_path=output_path,
|
||||
)
|
||||
)
|
||||
assert result["type"] == "WriteFiles"
|
||||
|
||||
async def __aenter__(self) -> "PagefindIndex":
|
||||
assert self._service is None
|
||||
assert self._index_id is None
|
||||
return await self._start()
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: Optional[Any],
|
||||
exc_value: Optional[Any],
|
||||
traceback: Optional[Any],
|
||||
) -> None:
|
||||
if self._service is None:
|
||||
return
|
||||
if self._index_id is None:
|
||||
return
|
||||
if exc_type is None:
|
||||
await self.write_files()
|
||||
await self._service.close()
|
||||
@@ -0,0 +1,170 @@
|
||||
from enum import Enum
|
||||
from typing import Dict, List, Literal, Optional, Sequence, TypedDict, Union
|
||||
|
||||
|
||||
class InternalRequestType(Enum):
|
||||
NEW_INDEX = "NewIndex"
|
||||
ADD_FILE = "AddFile"
|
||||
ADD_RECORD = "AddRecord"
|
||||
ADD_DIR = "AddDir"
|
||||
WRITE_FILES = "WriteFiles"
|
||||
GET_FILES = "GetFiles"
|
||||
DELETE_INDEX = "DeleteIndex"
|
||||
|
||||
|
||||
class InternalPagefindServiceConfig(TypedDict, total=False):
|
||||
# FIXME: document
|
||||
root_selector: Optional[str]
|
||||
exclude_selectors: Optional[Sequence[str]]
|
||||
force_language: Optional[str]
|
||||
verbose: Optional[bool]
|
||||
logfile: Optional[str]
|
||||
keep_index_url: Optional[bool]
|
||||
write_playground: Optional[bool]
|
||||
include_characters: Optional[str]
|
||||
|
||||
|
||||
class InternalNewIndexRequest(TypedDict):
|
||||
type: Literal["NewIndex"]
|
||||
config: Optional[InternalPagefindServiceConfig]
|
||||
|
||||
|
||||
class InternalAddFileRequest(TypedDict):
|
||||
type: Literal["AddFile"]
|
||||
index_id: int
|
||||
"""index_id must be positive."""
|
||||
file_path: Optional[str]
|
||||
url: Optional[str]
|
||||
file_contents: str
|
||||
|
||||
|
||||
class InternalAddRecordRequest(TypedDict):
|
||||
type: Literal["AddRecord"]
|
||||
index_id: int
|
||||
"""index_id must be positive."""
|
||||
url: str
|
||||
content: str
|
||||
language: str
|
||||
meta: Optional[Dict[str, str]]
|
||||
filters: Optional[Dict[str, List[str]]]
|
||||
sort: Optional[Dict[str, str]]
|
||||
|
||||
|
||||
class InternalAddDirRequest(TypedDict, total=False):
|
||||
type: Literal["AddDir"]
|
||||
index_id: int
|
||||
path: str # TODO: support Path
|
||||
glob: Optional[str]
|
||||
|
||||
|
||||
class InternalWriteFilesRequest(TypedDict, total=False):
|
||||
type: Literal["WriteFiles"]
|
||||
index_id: int
|
||||
"""index_id must be positive."""
|
||||
output_path: Optional[str]
|
||||
|
||||
|
||||
class InternalGetFilesRequest(TypedDict):
|
||||
type: Literal["GetFiles"]
|
||||
index_id: int
|
||||
"""index_id must be positive."""
|
||||
|
||||
|
||||
class InternalDeleteIndexRequest(TypedDict):
|
||||
type: Literal["DeleteIndex"]
|
||||
index_id: int
|
||||
"""index_id must be positive."""
|
||||
|
||||
|
||||
InternalRequestPayload = Union[
|
||||
InternalNewIndexRequest,
|
||||
InternalAddFileRequest,
|
||||
InternalAddRecordRequest,
|
||||
InternalAddDirRequest,
|
||||
InternalWriteFilesRequest,
|
||||
InternalGetFilesRequest,
|
||||
InternalDeleteIndexRequest,
|
||||
]
|
||||
|
||||
|
||||
class InternalServiceRequest(TypedDict):
|
||||
message_id: Optional[int]
|
||||
payload: InternalRequestPayload
|
||||
|
||||
|
||||
class InternalResponseType(Enum):
|
||||
NEW_INDEX = "NewIndex"
|
||||
INDEXED_FILE = "IndexedFile"
|
||||
INDEXED_DIR = "IndexedDir"
|
||||
WRITE_FILES = "WriteFiles"
|
||||
GET_FILES = "GetFiles"
|
||||
DELETE_INDEX = "DeleteIndex"
|
||||
ERROR = "Error"
|
||||
|
||||
|
||||
class InternalResponseError(TypedDict):
|
||||
type: Literal["Error"]
|
||||
message: str
|
||||
original_message: Optional[str]
|
||||
|
||||
|
||||
class InternalNewIndexResponse(TypedDict):
|
||||
type: Literal["NewIndex"]
|
||||
index_id: int
|
||||
|
||||
|
||||
class InternalIndexedFileResponse(TypedDict):
|
||||
type: Literal["IndexedFile"]
|
||||
page_word_count: int
|
||||
page_url: str
|
||||
page_meta: Dict[str, str]
|
||||
|
||||
|
||||
class InternalIndexedDirResponse(TypedDict):
|
||||
type: str
|
||||
page_count: int
|
||||
|
||||
|
||||
class InternalWriteFilesResponse(TypedDict):
|
||||
type: Literal["IndexedFile"]
|
||||
output_path: str
|
||||
|
||||
|
||||
class InternalSyntheticFile(TypedDict):
|
||||
path: str
|
||||
content: str
|
||||
|
||||
|
||||
class InternalDecodedFile(TypedDict):
|
||||
path: str
|
||||
content: bytes
|
||||
|
||||
|
||||
class InternalGetFilesResponse(TypedDict):
|
||||
type: Literal["GetFiles"]
|
||||
files: List[InternalSyntheticFile]
|
||||
|
||||
|
||||
class InternalDeleteIndexResponse(TypedDict):
|
||||
type: Literal["DeleteIndex"]
|
||||
|
||||
|
||||
InternalResponsePayload = Union[
|
||||
InternalNewIndexResponse,
|
||||
InternalIndexedFileResponse,
|
||||
InternalIndexedDirResponse,
|
||||
InternalWriteFilesResponse,
|
||||
InternalGetFilesResponse,
|
||||
InternalDeleteIndexResponse,
|
||||
]
|
||||
|
||||
|
||||
class InternalServiceResponse(TypedDict):
|
||||
message_id: Optional[int]
|
||||
payload: Union[InternalResponsePayload, InternalResponseError]
|
||||
|
||||
|
||||
class InternalResponseCallback(TypedDict, total=False):
|
||||
exception: Optional[Exception]
|
||||
err: Optional[InternalResponseError]
|
||||
result: Optional[InternalResponsePayload]
|
||||
Reference in new issue
Block a user