dualfroz
77465a50e7
Thread pages line up their posts as sibling <article> elements too, so in recall mode the container took every post along with its user card and quotes (katzen-forum.net, XenForo). The container is now used only when every entry is a teaser: its first heading links to a page of its own, which no other entry links to. Poster names come back along a thread or carry no link. On top of that it is never used on pages already detected as thread forums (DiscussionForumPosting).
729 lines
31 KiB
Python
729 lines
31 KiB
Python
# pylint:disable-msg=E0611,I1101
|
|
"""
|
|
Extraction configuration and processing functions.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import warnings
|
|
from configparser import ConfigParser
|
|
from copy import copy
|
|
from typing import Any
|
|
|
|
from lxml.etree import Element, XPath, _Element, strip_tags
|
|
from lxml.html import HtmlElement
|
|
|
|
# own
|
|
from .baseline import baseline, html2txt
|
|
from .deduplication import LRUCache, content_fingerprint, duplicate_test
|
|
from .external import compare_extraction, justext_rescue
|
|
from .htmlprocessing import (
|
|
build_html_output,
|
|
convert_tags,
|
|
prune_unwanted_nodes,
|
|
tree_cleaning,
|
|
)
|
|
from .main_extractor import _elem_text, extract_comments, extract_content
|
|
from .metadata import Document, extract_metadata
|
|
from .settings import DEFAULT_CONFIG, Extractor, use_config
|
|
from .utils import (
|
|
LANGID_FLAG,
|
|
HtmlInput,
|
|
check_html_lang,
|
|
language_filter,
|
|
load_html,
|
|
normalize_unicode,
|
|
)
|
|
from .xml import build_json_output, control_xml_output, delete_element, keeps_empty, xmltocsv, xmltotxt
|
|
from .xpaths import (
|
|
RAW_TREE_PRUNE_XPATH,
|
|
REMOVE_COMMENTS_AND_LISTS_XPATH,
|
|
REMOVE_COMMENTS_XPATH,
|
|
)
|
|
|
|
LOGGER = logging.getLogger(__name__)
|
|
|
|
# recall escalation (see trafilatura_sequence, stage 4): retry a short balanced extraction
|
|
# in recall mode. Calibrated as a set against the benchmark suite (own-bench/WMB/WCXB/AEB) —
|
|
# don't tune one value in isolation, and re-run the full suite after any change.
|
|
ESCALATION_MAX_LENGTH = 3000 # only consider extractions below this size
|
|
ESCALATION_PAGE_SHARE = 0.2 # ... covering less than this share of the page text
|
|
ESCALATION_ACCEPT_RATIO = 1.5 # accept the retry if it is this much longer
|
|
# justext tends to over-include (whole pricing tables, unrelated sections) rather than stop at
|
|
# page boundaries the way the rule-based retry does, so it needs a stricter bar than the
|
|
# retry's own 1.5x: swept 1.5-10x, 2.0x is the best balance across the full suite.
|
|
ESCALATION_JUSTEXT_RATIO = 2.0
|
|
|
|
TXT_FORMATS = {"markdown", "txt"}
|
|
_YAML_FIELDS = "title author url hostname description sitename date categories tags fingerprint id license".split()
|
|
|
|
# Metadata is emitted as a YAML-style Markdown header; values such as a title
|
|
# containing ": " (or a leading indicator, or a reserved word) otherwise produce
|
|
# invalid YAML or get reinterpreted as a non-string. See GH #814.
|
|
_YAML_RESERVED = frozenset({"true", "false", "yes", "no", "on", "off", "y", "n", "null", "none", "~"})
|
|
|
|
|
|
def _yaml_scalar(value: str) -> str:
|
|
"Render a metadata string as a plain or double-quoted YAML-safe scalar."
|
|
if (
|
|
value
|
|
and value == value.strip()
|
|
and value[0].isalpha()
|
|
and ": " not in value
|
|
and " #" not in value
|
|
and not value.endswith(":")
|
|
and value.lower() not in _YAML_RESERVED
|
|
and all(ch >= " " and ch != "\x7f" for ch in value)
|
|
):
|
|
return value
|
|
# a JSON string is always a valid, exactly round-tripping YAML double-quoted scalar
|
|
return json.dumps(value, ensure_ascii=False)
|
|
|
|
|
|
def determine_returnstring(document: Document, options: Extractor) -> str:
|
|
"""Convert XML tree to chosen format, clean the result and output it as a string"""
|
|
# XML (TEI) steps
|
|
if "xml" in options.format:
|
|
# last cleaning
|
|
for element in document.body.iter("*"):
|
|
if len(element) == 0 and not element.text and not element.tail and not keeps_empty(element):
|
|
delete_element(element, keep_tail=False)
|
|
# build output tree
|
|
returnstring = control_xml_output(document, options)
|
|
# CSV
|
|
elif options.format == "csv":
|
|
returnstring = xmltocsv(document, options.formatting)
|
|
# JSON
|
|
elif options.format == "json":
|
|
returnstring = build_json_output(document, options.with_metadata)
|
|
# HTML
|
|
elif options.format == "html":
|
|
returnstring = build_html_output(document, options.with_metadata)
|
|
# Markdown and TXT
|
|
else:
|
|
header = ""
|
|
if options.with_metadata:
|
|
# categories/tags are lists, their repr is a valid flow sequence
|
|
fields = "".join(
|
|
f"{attr}: {_yaml_scalar(value) if isinstance(value, str) else value}\n"
|
|
for attr in _YAML_FIELDS
|
|
if (value := getattr(document, attr))
|
|
)
|
|
header = f"---\n{fields}---\n"
|
|
returnstring = f"{header}{xmltotxt(document.body, options.formatting)}"
|
|
if document.commentsbody is not None:
|
|
returnstring = f"{returnstring}\n{xmltotxt(document.commentsbody, options.formatting)}".strip()
|
|
# normalize Unicode format (defaults to NFC)
|
|
return normalize_unicode(returnstring)
|
|
|
|
|
|
# matches "@type": "DiscussionForumPosting" or an @type array containing it, anchored to the
|
|
# key so it can't fire on the words appearing in ordinary prose (e.g. a description field)
|
|
_DISCUSSION_FORUM_POSTING_RE = re.compile(
|
|
r'"@type"\s*:\s*"DiscussionForumPosting"|"@type"\s*:\s*\[[^\]]*"DiscussionForumPosting"'
|
|
)
|
|
|
|
|
|
def _forum_thread_page(tree: HtmlElement) -> bool:
|
|
"""Detect a thread-forum page where posts live in the same containers
|
|
REMOVE_COMMENTS_XPATH would otherwise prune -- comments are content here, unlike on
|
|
a blog or article. Seeded by schema.org DiscussionForumPosting alone. Q&A forums
|
|
(StackExchange, schema.org QAPage) are deliberately not matched: their answers live
|
|
outside comment containers. Misses forums that don't emit DiscussionForumPosting
|
|
(e.g. old Reddit) -- an accepted gap. Meant as a reusable seed for a future page-type
|
|
router, not a one-off check.
|
|
"""
|
|
return any(
|
|
script.text and _DISCUSSION_FORUM_POSTING_RE.search(script.text)
|
|
for script in tree.iterfind('.//script[@type="application/ld+json"]')
|
|
)
|
|
|
|
|
|
def _prepare_tree(tree: HtmlElement, options: Extractor, url: str | None) -> tuple[HtmlElement, HtmlElement]:
|
|
"Clean and convert a raw tree, returning (converted, pre-conversion backup)."
|
|
cleaned = tree_cleaning(copy(tree), options)
|
|
backup = copy(cleaned)
|
|
cleaned = convert_tags(cleaned, options, url)
|
|
return cleaned, backup
|
|
|
|
|
|
def _extract_and_compare(
|
|
cleaned_tree: HtmlElement, cleaned_tree_backup: HtmlElement, tree: HtmlElement, options: Extractor, is_forum: bool
|
|
) -> tuple[_Element, str]:
|
|
"Cascade stages 1-2: main extractor, then the external comparison unless in fast mode."
|
|
postbody, temp_text = extract_content(cleaned_tree, options, is_forum)
|
|
if not options.fast:
|
|
postbody, temp_text = compare_extraction(cleaned_tree_backup, copy(tree), postbody, temp_text, options)
|
|
return postbody, temp_text
|
|
|
|
|
|
def _recall_retry(esc_tree: HtmlElement, r_options: Extractor, url: str | None, is_forum: bool) -> tuple[_Element, str]:
|
|
"""Stage-4 retry: re-run cascade stages 1-2 in recall mode on the escalation input
|
|
(arrives comment-pruned, or intact on a thread-forum where posts are content).
|
|
Deliberately no comment capture, no baseline (it already ran on the full page; on a
|
|
comment-pruned tree it only yields an indistinguishable boilerplate dump), no escalation."""
|
|
return _extract_and_compare(*_prepare_tree(esc_tree, r_options, url), esc_tree, r_options, is_forum)
|
|
|
|
|
|
def trafilatura_sequence(
|
|
tree: HtmlElement,
|
|
options: Extractor,
|
|
url: str | None = None,
|
|
) -> tuple[_Element, str, _Element, str]:
|
|
"""Prepare the raw tree (cleaning, tag conversion, comment handling), then execute the
|
|
standard cascade of extractors used by Trafilatura, each stage only engaging if the
|
|
previous one under-delivered:
|
|
1. main extractor (includes wild-text recovery for short documents)
|
|
2. comparison with external extractors (readability/justext), skipped in fast mode
|
|
3. baseline rescue on the original, uncleaned tree
|
|
4. recall escalation, if the result still covers little of the page: stages 1-2 re-run
|
|
in recall mode (_recall_retry), plus a justext candidate tried alongside (a different
|
|
algorithm, not just stricter rules, so it reaches content the rule-based retry cannot)
|
|
Returns the body and comments elements with their text.
|
|
|
|
Internal helper: its signature and 4-tuple return are not a stable API — call
|
|
``bare_extraction``/``extract`` instead.
|
|
"""
|
|
is_forum = _forum_thread_page(tree)
|
|
# raw-tree prune so the external extractors inherit it too: readability would otherwise
|
|
# pick the longest appended article over the real one
|
|
tree = prune_unwanted_nodes(tree, RAW_TREE_PRUNE_XPATH)
|
|
# comments off: prune the raw tree so all stages inherit it, lists are still ol/ul here
|
|
if not options.comments and (options.focus == "precision" or not is_forum):
|
|
tree = prune_unwanted_nodes(tree, REMOVE_COMMENTS_AND_LISTS_XPATH)
|
|
cleaned_tree, cleaned_tree_backup = _prepare_tree(tree, options, url)
|
|
|
|
commentsbody, temp_comments = Element("body"), ""
|
|
forum_posts = None
|
|
if options.comments:
|
|
commentsbody, temp_comments, cleaned_tree = extract_comments(cleaned_tree, options)
|
|
if temp_comments and is_forum:
|
|
# thread-forum: the "comments" are the posts -> route into the body (backup predates
|
|
# capture); keep the capture aside, salvaged below if the cascade drops the posts
|
|
forum_posts = commentsbody
|
|
commentsbody, temp_comments = Element("body"), ""
|
|
cleaned_tree = convert_tags(copy(cleaned_tree_backup), options, url)
|
|
if options.focus == "precision" and not is_forum:
|
|
# NOT redundant with the raw-tree prune above: this runs POST-conversion, where lists are <list>
|
|
# and match the xpath's self::list (dl lists, disqus ids, and all lists if comments are on)
|
|
cleaned_tree = prune_unwanted_nodes(cleaned_tree, REMOVE_COMMENTS_XPATH)
|
|
|
|
postbody, temp_text = _extract_and_compare(cleaned_tree, cleaned_tree_backup, tree, options, is_forum)
|
|
|
|
# 3. rescue: baseline on the original tree, accepted only if it adds text (#896)
|
|
if len(temp_text) < options.min_extracted_size and options.focus != "precision":
|
|
b_body, b_text, b_len = baseline(tree) # baseline copies element inputs
|
|
LOGGER.debug("non-clean extracted length: %s (extraction)", b_len)
|
|
if b_len > len(temp_text):
|
|
postbody, temp_text = b_body, b_text
|
|
forum_posts = None # the dump saw the whole page: missing posts are boilerplate, not lost
|
|
|
|
# 4. recall escalation: a short extraction covering little of the page suggests
|
|
# under-extraction (non-article layout) — retry in recall mode, keep if clearly bigger.
|
|
# NOTE: the page measure html2txt(tree) is coupled to BASIC_CLEAN_XPATH — see settings.py.
|
|
if (
|
|
options.focus == "balanced"
|
|
and 0 < len(temp_text) < ESCALATION_MAX_LENGTH
|
|
and len(temp_text) < ESCALATION_PAGE_SHARE * len(html2txt(tree))
|
|
):
|
|
# a copy so a shared Extractor never leaks the "recall" focus back to the caller
|
|
r_options = copy(options)
|
|
r_options.focus = "recall"
|
|
# strip comment containers from the escalation input (dup risk if captured, reader comments if not),
|
|
# ol/ul comment lists only if captured (#881). Not on a thread-forum, where the retry rescues the posts.
|
|
# Comments off: already pruned
|
|
esc_tree = (
|
|
tree
|
|
if is_forum or not options.comments
|
|
else prune_unwanted_nodes(copy(tree), REMOVE_COMMENTS_AND_LISTS_XPATH if temp_comments else REMOVE_COMMENTS_XPATH)
|
|
)
|
|
r_text = ""
|
|
try:
|
|
r_body, r_text = _recall_retry(esc_tree, r_options, url, is_forum)
|
|
except Exception as err: # pragma: no cover
|
|
LOGGER.warning("recall retry failed: %s %s", err, url)
|
|
# justext reaches div-buried content the rule retry misses (gated: ungated regressed
|
|
# own-fallback). No region scoping of its own -> esc_tree is comment-pruned above
|
|
j_text = ""
|
|
if not options.fast:
|
|
try:
|
|
j_body, j_text = justext_rescue(copy(esc_tree), options)
|
|
except Exception as err: # pragma: no cover
|
|
LOGGER.warning("justext candidate failed: %s %s", err, url)
|
|
|
|
# floor on the retry: a result below the pipeline's own minimum is noise (the retry's
|
|
# former internal baseline used to displace such outputs). cookie/consent banners justext
|
|
# could pick up are pruned from its input in basic_cleaning. An accepted candidate saw the
|
|
# full page, so its exclusions are deliberate -> drop the forum-post salvage.
|
|
len_text, r_len, j_len = len(temp_text), len(r_text), len(j_text)
|
|
if j_len > r_len and j_len > ESCALATION_JUSTEXT_RATIO * len_text:
|
|
postbody, temp_text, forum_posts = j_body, j_text, None
|
|
elif r_len >= options.min_extracted_size and r_len > ESCALATION_ACCEPT_RATIO * len_text:
|
|
postbody, temp_text, forum_posts = r_body, r_text, None
|
|
|
|
if forum_posts is not None:
|
|
# a gate (escalation length, precision) blocked the cascade from restoring the posts:
|
|
# append the ones missing from the body
|
|
existing = "\n".join(filter(None, (_elem_text(el) for el in postbody)))
|
|
salvaged = [el for el in forum_posts if (t := _elem_text(el)) and t not in existing]
|
|
if salvaged:
|
|
LOGGER.debug("thread-forum salvage: %s captured posts appended to the body", len(salvaged))
|
|
postbody.extend(salvaged)
|
|
temp_text = " ".join(postbody.itertext()).strip()
|
|
|
|
return postbody, temp_text, commentsbody, temp_comments
|
|
|
|
|
|
def bare_extraction(
|
|
filecontent: HtmlInput,
|
|
url: str | None = None,
|
|
fast: bool = False,
|
|
no_fallback: bool = False,
|
|
favor_precision: bool = False,
|
|
favor_recall: bool = False,
|
|
include_comments: bool = True,
|
|
output_format: str = "python",
|
|
target_language: str | None = None,
|
|
include_tables: bool = True,
|
|
include_images: bool = False,
|
|
include_formatting: bool | None = None,
|
|
include_links: bool = False,
|
|
deduplicate: bool | LRUCache = False,
|
|
date_extraction_params: dict[str, Any] | None = None,
|
|
with_metadata: bool = False,
|
|
only_with_metadata: bool = False,
|
|
max_tree_size: int | None = None,
|
|
url_blacklist: set[str] | None = None,
|
|
author_blacklist: set[str] | None = None,
|
|
as_dict: bool = False,
|
|
prune_xpath: str | list[str] | None = None,
|
|
config: ConfigParser = DEFAULT_CONFIG,
|
|
options: Extractor | None = None,
|
|
) -> Document | dict[str, Any] | None:
|
|
"""Internal function for text extraction returning bare Python variables.
|
|
|
|
Args:
|
|
filecontent: HTML code as string.
|
|
url: URL of the webpage.
|
|
fast: Use faster heuristics and skip backup extraction.
|
|
no_fallback: Deprecated, use "fast" instead.
|
|
favor_precision: prefer less text but correct extraction.
|
|
favor_recall: prefer more text even when unsure.
|
|
include_comments: Extract comments along with the main text.
|
|
output_format: Define an output format, Python being the default
|
|
and the interest of this internal function.
|
|
Other values: "csv", "html", "json", "markdown", "txt", "xml", and "xmltei".
|
|
target_language: Define a language to discard invalid documents (ISO 639-1 format).
|
|
include_tables: Take into account information within the HTML <table> element.
|
|
include_images: Take images into account.
|
|
include_formatting: Keep structural elements related to formatting
|
|
(kept in XML, rendered as markdown for text formats; ignored for JSON).
|
|
include_links: Keep links along with their targets.
|
|
deduplicate: Remove duplicate segments and documents.
|
|
Accepts an LRUCache instance as dedicated, scoped cache.
|
|
date_extraction_params: Provide extraction parameters to htmldate as dict().
|
|
with_metadata: Extract metadata fields and add them to the output.
|
|
only_with_metadata: Only keep documents featuring all essential metadata
|
|
(date, title, url).
|
|
url_blacklist: Provide a blacklist of URLs as set() to filter out documents.
|
|
author_blacklist: Provide a blacklist of Author Names as set() to filter out authors.
|
|
as_dict: Deprecated, use the .as_dict() method instead.
|
|
prune_xpath: Provide an XPath expression to prune the tree before extraction.
|
|
can be str or list of str.
|
|
config: Directly provide a configparser configuration.
|
|
options: Directly provide a whole extractor configuration.
|
|
|
|
Returns:
|
|
A Python dict() containing all the extracted information or None.
|
|
|
|
Raises:
|
|
ValueError: Extraction problem.
|
|
|
|
Note:
|
|
Low-level primitive: returns a Document with an unserialized .body tree; tei_validation only applies when serializing via extract().
|
|
In the default balanced mode, a short extraction covering little of the page is automatically retried with recall settings.
|
|
"""
|
|
|
|
# deprecations: stacklevel=3 → user → bare_extraction → _check_deprecation
|
|
fast = _check_deprecation(
|
|
fast,
|
|
no_fallback=no_fallback,
|
|
as_dict=as_dict,
|
|
max_tree_size=max_tree_size,
|
|
stacklevel=3,
|
|
)
|
|
|
|
# regroup extraction options
|
|
if not options or not isinstance(options, Extractor):
|
|
options = Extractor(
|
|
config=config,
|
|
output_format=output_format,
|
|
fast=fast,
|
|
precision=favor_precision,
|
|
recall=favor_recall,
|
|
comments=include_comments,
|
|
formatting=include_formatting,
|
|
links=include_links,
|
|
images=include_images,
|
|
tables=include_tables,
|
|
dedup=deduplicate,
|
|
lang=target_language,
|
|
url=url,
|
|
with_metadata=with_metadata,
|
|
only_with_metadata=only_with_metadata,
|
|
author_blacklist=author_blacklist,
|
|
url_blacklist=url_blacklist,
|
|
date_params=date_extraction_params,
|
|
)
|
|
|
|
try:
|
|
# load the HTML tree
|
|
tree = load_html(filecontent, options.max_file_size)
|
|
if tree is None:
|
|
LOGGER.error("empty HTML tree: %s", url)
|
|
raise ValueError
|
|
if tree is filecontent:
|
|
tree = copy(tree)
|
|
|
|
# quick and dirty HTML lang check
|
|
if options.lang and (options.fast or not LANGID_FLAG):
|
|
if check_html_lang(tree, options.lang) is False:
|
|
LOGGER.error("wrong HTML meta language: %s", options.source)
|
|
raise ValueError
|
|
|
|
# extract metadata if necessary
|
|
if options.with_metadata:
|
|
document = extract_metadata(
|
|
tree,
|
|
options.url,
|
|
options.date_params,
|
|
options.fast,
|
|
options.author_blacklist,
|
|
)
|
|
|
|
# cut short if extracted URL in blacklist
|
|
if document.url in options.url_blacklist:
|
|
LOGGER.warning("blacklisted URL: %s", document.url)
|
|
raise ValueError
|
|
|
|
# cut short if core elements are missing
|
|
if options.only_with_metadata and not (document.date and document.title and document.url):
|
|
LOGGER.error("no metadata: %s", options.source)
|
|
raise ValueError
|
|
|
|
else:
|
|
document = Document()
|
|
|
|
# prune all xpath expressions that user specified
|
|
# no backup as this is unetre full control of the user
|
|
if prune_xpath is not None:
|
|
if isinstance(prune_xpath, str):
|
|
prune_xpath = [prune_xpath]
|
|
tree = prune_unwanted_nodes(tree, [XPath(x) for x in prune_xpath])
|
|
|
|
postbody, temp_text, commentsbody, temp_comments = trafilatura_sequence(tree, options, options.url or document.url)
|
|
len_text, len_comments = len(temp_text), len(temp_comments)
|
|
|
|
# tree size sanity check
|
|
if options.max_tree_size:
|
|
# strip tags
|
|
if len(postbody) > options.max_tree_size:
|
|
LOGGER.debug("output tree too long: %s", len(postbody))
|
|
strip_tags(postbody, "hi")
|
|
# still too long, raise an error
|
|
if len(postbody) > options.max_tree_size:
|
|
LOGGER.debug(
|
|
"output tree too long: %s, discarding %s",
|
|
len(postbody),
|
|
options.source,
|
|
)
|
|
raise ValueError
|
|
# size checks
|
|
if len_text < options.min_output_size and len_comments < options.min_output_comm_size:
|
|
LOGGER.debug(
|
|
"text and comments not long enough: %s %s %s",
|
|
len_text,
|
|
len_comments,
|
|
options.source,
|
|
)
|
|
raise ValueError
|
|
|
|
# check duplicates at body level
|
|
if options.dedup and duplicate_test(postbody, options) is True:
|
|
LOGGER.debug("discarding duplicate document: %s", options.source)
|
|
raise ValueError
|
|
|
|
# sanity check on language
|
|
if options.lang:
|
|
is_not_target_lang, document = language_filter(temp_text, temp_comments, options.lang, document)
|
|
if is_not_target_lang is True:
|
|
LOGGER.debug("wrong language: %s", options.source)
|
|
raise ValueError
|
|
|
|
except (TypeError, ValueError):
|
|
LOGGER.warning("discarding data: %s", options.source)
|
|
return None
|
|
|
|
# special case: python variables
|
|
if options.format == "python":
|
|
document.text = xmltotxt(postbody, options.formatting)
|
|
if options.comments:
|
|
document.comments = xmltotxt(commentsbody, options.formatting)
|
|
document.commentsbody = commentsbody
|
|
document.raw_text = document.text
|
|
else:
|
|
document.raw_text, document.commentsbody = temp_text, commentsbody
|
|
document.body = postbody
|
|
|
|
return document if not as_dict else document.as_dict()
|
|
|
|
|
|
def extract(
|
|
filecontent: HtmlInput,
|
|
url: str | None = None,
|
|
record_id: str | None = None,
|
|
fast: bool = False,
|
|
no_fallback: bool = False,
|
|
favor_precision: bool = False,
|
|
favor_recall: bool = False,
|
|
include_comments: bool = True,
|
|
output_format: str = "txt",
|
|
tei_validation: bool = False,
|
|
target_language: str | None = None,
|
|
include_tables: bool = True,
|
|
include_images: bool = False,
|
|
include_formatting: bool | None = None,
|
|
include_links: bool = False,
|
|
deduplicate: bool | LRUCache = False,
|
|
date_extraction_params: dict[str, Any] | None = None,
|
|
with_metadata: bool = False,
|
|
only_with_metadata: bool = False,
|
|
max_tree_size: int | None = None,
|
|
url_blacklist: set[str] | None = None,
|
|
author_blacklist: set[str] | None = None,
|
|
settingsfile: str | None = None,
|
|
prune_xpath: str | list[str] | None = None,
|
|
config: ConfigParser = DEFAULT_CONFIG,
|
|
options: Extractor | None = None,
|
|
) -> str | None:
|
|
"""Main function exposed by the package:
|
|
Wrapper for text extraction and conversion to chosen output format.
|
|
|
|
Args:
|
|
filecontent: HTML code as string.
|
|
url: URL of the webpage.
|
|
record_id: Add an ID to the metadata.
|
|
fast: Use faster heuristics and skip backup extraction.
|
|
no_fallback: Deprecated, use "fast" instead.
|
|
favor_precision: prefer less text but correct extraction.
|
|
favor_recall: when unsure, prefer more text.
|
|
include_comments: Extract comments along with the main text.
|
|
output_format: Define an output format:
|
|
"csv", "html", "json", "markdown", "txt", "xml", and "xmltei".
|
|
tei_validation: Validate the XML-TEI output with respect to the TEI standard.
|
|
target_language: Define a language to discard invalid documents (ISO 639-1 format).
|
|
include_tables: Take into account information within the HTML <table> element.
|
|
include_images: Take images into account.
|
|
include_formatting: Keep structural elements related to formatting
|
|
(kept in XML, rendered as markdown for text formats; ignored for JSON).
|
|
include_links: Keep links along with their targets.
|
|
deduplicate: Remove duplicate segments and documents.
|
|
Accepts an LRUCache instance as dedicated, scoped cache.
|
|
date_extraction_params: Provide extraction parameters to htmldate as dict().
|
|
with_metadata: Extract metadata fields and add them to the output.
|
|
only_with_metadata: Only keep documents featuring all essential metadata
|
|
(date, title, url).
|
|
url_blacklist: Provide a blacklist of URLs as set() to filter out documents.
|
|
author_blacklist: Provide a blacklist of Author Names as set() to filter out authors.
|
|
settingsfile: Use a configuration file to override the standard settings.
|
|
prune_xpath: Provide an XPath expression to prune the tree before extraction.
|
|
can be str or list of str.
|
|
config: Directly provide a configparser configuration.
|
|
options: Directly provide a whole extractor configuration.
|
|
|
|
Returns:
|
|
A string in the desired format or None.
|
|
|
|
"""
|
|
# must stay the first statement: a signature drift raises TypeError
|
|
document = _internal_extraction(**locals())
|
|
return document.text if document is not None else None
|
|
|
|
|
|
def extract_with_metadata(
|
|
filecontent: HtmlInput,
|
|
url: str | None = None,
|
|
record_id: str | None = None,
|
|
fast: bool = False,
|
|
favor_precision: bool = False,
|
|
favor_recall: bool = False,
|
|
include_comments: bool = True,
|
|
output_format: str = "txt",
|
|
tei_validation: bool = False,
|
|
target_language: str | None = None,
|
|
include_tables: bool = True,
|
|
include_images: bool = False,
|
|
include_formatting: bool | None = None,
|
|
include_links: bool = False,
|
|
deduplicate: bool | LRUCache = False,
|
|
date_extraction_params: dict[str, Any] | None = None,
|
|
url_blacklist: set[str] | None = None,
|
|
author_blacklist: set[str] | None = None,
|
|
settingsfile: str | None = None,
|
|
prune_xpath: str | list[str] | None = None,
|
|
config: ConfigParser = DEFAULT_CONFIG,
|
|
options: Extractor | None = None,
|
|
) -> Document | None:
|
|
"""Main function exposed by the package:
|
|
Wrapper for text extraction and conversion to chosen output format.
|
|
This method also returns document metadata.
|
|
|
|
Args:
|
|
filecontent: HTML code as string.
|
|
url: URL of the webpage.
|
|
record_id: Add an ID to the metadata.
|
|
fast: Use faster heuristics and skip backup extraction.
|
|
favor_precision: prefer less text but correct extraction.
|
|
favor_recall: when unsure, prefer more text.
|
|
include_comments: Extract comments along with the main text.
|
|
output_format: Define an output format:
|
|
"csv", "html", "json", "markdown", "txt", "xml", and "xmltei".
|
|
tei_validation: Validate the XML-TEI output with respect to the TEI standard.
|
|
target_language: Define a language to discard invalid documents (ISO 639-1 format).
|
|
include_tables: Take into account information within the HTML <table> element.
|
|
include_images: Take images into account.
|
|
include_formatting: Keep structural elements related to formatting
|
|
(kept in XML, rendered as markdown for text formats; ignored for JSON).
|
|
include_links: Keep links along with their targets.
|
|
deduplicate: Remove duplicate segments and documents.
|
|
Accepts an LRUCache instance as dedicated, scoped cache.
|
|
date_extraction_params: Provide extraction parameters to htmldate as dict().
|
|
url_blacklist: Provide a blacklist of URLs as set() to filter out documents.
|
|
author_blacklist: Provide a blacklist of Author Names as set() to filter out authors.
|
|
settingsfile: Use a configuration file to override the standard settings.
|
|
prune_xpath: Provide an XPath expression to prune the tree before extraction.
|
|
can be str or list of str.
|
|
config: Directly provide a configparser configuration.
|
|
options: Directly provide a whole extractor configuration.
|
|
|
|
Returns:
|
|
Document metadata with content string in the desired format or None.
|
|
"""
|
|
return _internal_extraction(**locals(), with_metadata=True)
|
|
|
|
|
|
def _check_deprecation(
|
|
fast: bool = False,
|
|
*,
|
|
no_fallback: bool = False,
|
|
as_dict: bool = False,
|
|
max_tree_size: int | None = None,
|
|
stacklevel: int = 2,
|
|
) -> bool:
|
|
"""Check deprecated params and return the effective "fast" flag."""
|
|
if no_fallback:
|
|
warnings.warn(
|
|
'"no_fallback" will be removed, use "fast" instead',
|
|
DeprecationWarning,
|
|
stacklevel=stacklevel,
|
|
)
|
|
if as_dict:
|
|
warnings.warn(
|
|
'"as_dict" will be removed, use the .as_dict() method instead',
|
|
DeprecationWarning,
|
|
stacklevel=stacklevel,
|
|
)
|
|
if max_tree_size:
|
|
raise ValueError('"max_tree_size" will be removed, use settings.cfg instead')
|
|
return fast or no_fallback
|
|
|
|
|
|
def _internal_extraction(
|
|
filecontent: HtmlInput,
|
|
url: str | None = None,
|
|
record_id: str | None = None,
|
|
fast: bool = False,
|
|
no_fallback: bool = False,
|
|
favor_precision: bool = False,
|
|
favor_recall: bool = False,
|
|
include_comments: bool = True,
|
|
output_format: str = "txt",
|
|
tei_validation: bool = False,
|
|
target_language: str | None = None,
|
|
include_tables: bool = True,
|
|
include_images: bool = False,
|
|
include_formatting: bool | None = None,
|
|
include_links: bool = False,
|
|
deduplicate: bool | LRUCache = False,
|
|
date_extraction_params: dict[str, Any] | None = None,
|
|
with_metadata: bool = False,
|
|
only_with_metadata: bool = False,
|
|
max_tree_size: int | None = None,
|
|
url_blacklist: set[str] | None = None,
|
|
author_blacklist: set[str] | None = None,
|
|
settingsfile: str | None = None,
|
|
prune_xpath: str | list[str] | None = None,
|
|
config: ConfigParser = DEFAULT_CONFIG,
|
|
options: Extractor | None = None,
|
|
) -> Document | None:
|
|
"""Internal method to do the extraction"""
|
|
# stacklevel=4 → user → extract → _internal_extraction → _check_deprecation
|
|
fast = _check_deprecation(
|
|
fast,
|
|
no_fallback=no_fallback,
|
|
as_dict=False,
|
|
max_tree_size=max_tree_size,
|
|
stacklevel=4,
|
|
)
|
|
|
|
# regroup extraction options
|
|
if not options or not isinstance(options, Extractor):
|
|
options = Extractor(
|
|
config=use_config(settingsfile) if settingsfile else config,
|
|
output_format=output_format,
|
|
fast=fast,
|
|
precision=favor_precision,
|
|
recall=favor_recall,
|
|
comments=include_comments,
|
|
formatting=include_formatting,
|
|
links=include_links,
|
|
images=include_images,
|
|
tables=include_tables,
|
|
dedup=deduplicate,
|
|
lang=target_language,
|
|
url=url,
|
|
with_metadata=with_metadata,
|
|
only_with_metadata=only_with_metadata,
|
|
tei_validation=tei_validation,
|
|
author_blacklist=author_blacklist,
|
|
url_blacklist=url_blacklist,
|
|
date_params=date_extraction_params,
|
|
)
|
|
|
|
# extraction
|
|
document = bare_extraction(
|
|
filecontent,
|
|
options=options,
|
|
as_dict=False,
|
|
prune_xpath=prune_xpath,
|
|
)
|
|
|
|
# post-processing
|
|
if not document or not isinstance(document, Document):
|
|
return None
|
|
|
|
# TXT formats only output the ID and fingerprint in their metadata header
|
|
if options.format not in TXT_FORMATS or options.with_metadata:
|
|
# control output
|
|
if options.format == "python":
|
|
raise ValueError("'python' format only usable in bare_extraction() function")
|
|
# add record ID to metadata
|
|
document.id = record_id
|
|
# calculate fingerprint
|
|
if document.raw_text is not None:
|
|
document.fingerprint = content_fingerprint(str(document.title) + " " + str(document.raw_text))
|
|
|
|
document.text = determine_returnstring(document, options)
|
|
return document
|