From 03e823a030590bd625ab99258629815a22029c51 Mon Sep 17 00:00:00 2001 From: upstream Date: Fri, 2 Oct 2026 17:03:46 +0200 Subject: [PATCH] Upstream adbar/trafilatura state of the touched files before 1377ec86cc --- trafilatura/core.py | 728 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 728 insertions(+) create mode 100644 trafilatura/core.py diff --git a/trafilatura/core.py b/trafilatura/core.py new file mode 100644 index 0000000..50804bc --- /dev/null +++ b/trafilatura/core.py @@ -0,0 +1,728 @@ +# pylint:disable-msg=E0611,I1101 +""" +Extraction configuration and processing functions. +""" + +import json +import logging +import re +import warnings +from configparser import ConfigParser +from copy import copy +from typing import Any + +from lxml.etree import Element, XPath, _Element, strip_tags +from lxml.html import HtmlElement + +# own +from .baseline import baseline, html2txt +from .deduplication import LRUCache, content_fingerprint, duplicate_test +from .external import compare_extraction, justext_rescue +from .htmlprocessing import ( + build_html_output, + convert_tags, + prune_unwanted_nodes, + tree_cleaning, +) +from .main_extractor import _elem_text, extract_comments, extract_content +from .metadata import Document, extract_metadata +from .settings import DEFAULT_CONFIG, Extractor, use_config +from .utils import ( + LANGID_FLAG, + HtmlInput, + check_html_lang, + language_filter, + load_html, + normalize_unicode, +) +from .xml import build_json_output, control_xml_output, delete_element, keeps_empty, xmltocsv, xmltotxt +from .xpaths import ( + RAW_TREE_PRUNE_XPATH, + REMOVE_COMMENTS_AND_LISTS_XPATH, + REMOVE_COMMENTS_XPATH, +) + +LOGGER = logging.getLogger(__name__) + +# recall escalation (see trafilatura_sequence, stage 4): retry a short balanced extraction +# in recall mode. Calibrated as a set against the benchmark suite (own-bench/WMB/WCXB/AEB) — +# don't tune one value in isolation, and re-run the full suite after any change. +ESCALATION_MAX_LENGTH = 3000 # only consider extractions below this size +ESCALATION_PAGE_SHARE = 0.2 # ... covering less than this share of the page text +ESCALATION_ACCEPT_RATIO = 1.5 # accept the retry if it is this much longer +# justext tends to over-include (whole pricing tables, unrelated sections) rather than stop at +# page boundaries the way the rule-based retry does, so it needs a stricter bar than the +# retry's own 1.5x: swept 1.5-10x, 2.0x is the best balance across the full suite. +ESCALATION_JUSTEXT_RATIO = 2.0 + +TXT_FORMATS = {"markdown", "txt"} +_YAML_FIELDS = "title author url hostname description sitename date categories tags fingerprint id license".split() + +# Metadata is emitted as a YAML-style Markdown header; values such as a title +# containing ": " (or a leading indicator, or a reserved word) otherwise produce +# invalid YAML or get reinterpreted as a non-string. See GH #814. +_YAML_RESERVED = frozenset({"true", "false", "yes", "no", "on", "off", "y", "n", "null", "none", "~"}) + + +def _yaml_scalar(value: str) -> str: + "Render a metadata string as a plain or double-quoted YAML-safe scalar." + if ( + value + and value == value.strip() + and value[0].isalpha() + and ": " not in value + and " #" not in value + and not value.endswith(":") + and value.lower() not in _YAML_RESERVED + and all(ch >= " " and ch != "\x7f" for ch in value) + ): + return value + # a JSON string is always a valid, exactly round-tripping YAML double-quoted scalar + return json.dumps(value, ensure_ascii=False) + + +def determine_returnstring(document: Document, options: Extractor) -> str: + """Convert XML tree to chosen format, clean the result and output it as a string""" + # XML (TEI) steps + if "xml" in options.format: + # last cleaning + for element in document.body.iter("*"): + if len(element) == 0 and not element.text and not element.tail and not keeps_empty(element): + delete_element(element, keep_tail=False) + # build output tree + returnstring = control_xml_output(document, options) + # CSV + elif options.format == "csv": + returnstring = xmltocsv(document, options.formatting) + # JSON + elif options.format == "json": + returnstring = build_json_output(document, options.with_metadata) + # HTML + elif options.format == "html": + returnstring = build_html_output(document, options.with_metadata) + # Markdown and TXT + else: + header = "" + if options.with_metadata: + # categories/tags are lists, their repr is a valid flow sequence + fields = "".join( + f"{attr}: {_yaml_scalar(value) if isinstance(value, str) else value}\n" + for attr in _YAML_FIELDS + if (value := getattr(document, attr)) + ) + header = f"---\n{fields}---\n" + returnstring = f"{header}{xmltotxt(document.body, options.formatting)}" + if document.commentsbody is not None: + returnstring = f"{returnstring}\n{xmltotxt(document.commentsbody, options.formatting)}".strip() + # normalize Unicode format (defaults to NFC) + return normalize_unicode(returnstring) + + +# matches "@type": "DiscussionForumPosting" or an @type array containing it, anchored to the +# key so it can't fire on the words appearing in ordinary prose (e.g. a description field) +_DISCUSSION_FORUM_POSTING_RE = re.compile( + r'"@type"\s*:\s*"DiscussionForumPosting"|"@type"\s*:\s*\[[^\]]*"DiscussionForumPosting"' +) + + +def _forum_thread_page(tree: HtmlElement) -> bool: + """Detect a thread-forum page where posts live in the same containers + REMOVE_COMMENTS_XPATH would otherwise prune -- comments are content here, unlike on + a blog or article. Seeded by schema.org DiscussionForumPosting alone. Q&A forums + (StackExchange, schema.org QAPage) are deliberately not matched: their answers live + outside comment containers. Misses forums that don't emit DiscussionForumPosting + (e.g. old Reddit) -- an accepted gap. Meant as a reusable seed for a future page-type + router, not a one-off check. + """ + return any( + script.text and _DISCUSSION_FORUM_POSTING_RE.search(script.text) + for script in tree.iterfind('.//script[@type="application/ld+json"]') + ) + + +def _prepare_tree(tree: HtmlElement, options: Extractor, url: str | None) -> tuple[HtmlElement, HtmlElement]: + "Clean and convert a raw tree, returning (converted, pre-conversion backup)." + cleaned = tree_cleaning(copy(tree), options) + backup = copy(cleaned) + cleaned = convert_tags(cleaned, options, url) + return cleaned, backup + + +def _extract_and_compare( + cleaned_tree: HtmlElement, cleaned_tree_backup: HtmlElement, tree: HtmlElement, options: Extractor +) -> tuple[_Element, str]: + "Cascade stages 1-2: main extractor, then the external comparison unless in fast mode." + postbody, temp_text = extract_content(cleaned_tree, options) + if not options.fast: + postbody, temp_text = compare_extraction(cleaned_tree_backup, copy(tree), postbody, temp_text, options) + return postbody, temp_text + + +def _recall_retry(esc_tree: HtmlElement, r_options: Extractor, url: str | None) -> tuple[_Element, str]: + """Stage-4 retry: re-run cascade stages 1-2 in recall mode on the escalation input + (arrives comment-pruned, or intact on a thread-forum where posts are content). + Deliberately no comment capture, no baseline (it already ran on the full page; on a + comment-pruned tree it only yields an indistinguishable boilerplate dump), no escalation.""" + return _extract_and_compare(*_prepare_tree(esc_tree, r_options, url), esc_tree, r_options) + + +def trafilatura_sequence( + tree: HtmlElement, + options: Extractor, + url: str | None = None, +) -> tuple[_Element, str, _Element, str]: + """Prepare the raw tree (cleaning, tag conversion, comment handling), then execute the + standard cascade of extractors used by Trafilatura, each stage only engaging if the + previous one under-delivered: + 1. main extractor (includes wild-text recovery for short documents) + 2. comparison with external extractors (readability/justext), skipped in fast mode + 3. baseline rescue on the original, uncleaned tree + 4. recall escalation, if the result still covers little of the page: stages 1-2 re-run + in recall mode (_recall_retry), plus a justext candidate tried alongside (a different + algorithm, not just stricter rules, so it reaches content the rule-based retry cannot) + Returns the body and comments elements with their text. + + Internal helper: its signature and 4-tuple return are not a stable API — call + ``bare_extraction``/``extract`` instead. + """ + is_forum = _forum_thread_page(tree) + # raw-tree prune so the external extractors inherit it too: readability would otherwise + # pick the longest appended article over the real one + tree = prune_unwanted_nodes(tree, RAW_TREE_PRUNE_XPATH) + # comments off: prune the raw tree so all stages inherit it, lists are still ol/ul here + if not options.comments and (options.focus == "precision" or not is_forum): + tree = prune_unwanted_nodes(tree, REMOVE_COMMENTS_AND_LISTS_XPATH) + cleaned_tree, cleaned_tree_backup = _prepare_tree(tree, options, url) + + commentsbody, temp_comments = Element("body"), "" + forum_posts = None + if options.comments: + commentsbody, temp_comments, cleaned_tree = extract_comments(cleaned_tree, options) + if temp_comments and is_forum: + # thread-forum: the "comments" are the posts -> route into the body (backup predates + # capture); keep the capture aside, salvaged below if the cascade drops the posts + forum_posts = commentsbody + commentsbody, temp_comments = Element("body"), "" + cleaned_tree = convert_tags(copy(cleaned_tree_backup), options, url) + if options.focus == "precision" and not is_forum: + # NOT redundant with the raw-tree prune above: this runs POST-conversion, where lists are + # and match the xpath's self::list (dl lists, disqus ids, and all lists if comments are on) + cleaned_tree = prune_unwanted_nodes(cleaned_tree, REMOVE_COMMENTS_XPATH) + + postbody, temp_text = _extract_and_compare(cleaned_tree, cleaned_tree_backup, tree, options) + + # 3. rescue: baseline on the original tree, accepted only if it adds text (#896) + if len(temp_text) < options.min_extracted_size and options.focus != "precision": + b_body, b_text, b_len = baseline(tree) # baseline copies element inputs + LOGGER.debug("non-clean extracted length: %s (extraction)", b_len) + if b_len > len(temp_text): + postbody, temp_text = b_body, b_text + forum_posts = None # the dump saw the whole page: missing posts are boilerplate, not lost + + # 4. recall escalation: a short extraction covering little of the page suggests + # under-extraction (non-article layout) — retry in recall mode, keep if clearly bigger. + # NOTE: the page measure html2txt(tree) is coupled to BASIC_CLEAN_XPATH — see settings.py. + if ( + options.focus == "balanced" + and 0 < len(temp_text) < ESCALATION_MAX_LENGTH + and len(temp_text) < ESCALATION_PAGE_SHARE * len(html2txt(tree)) + ): + # a copy so a shared Extractor never leaks the "recall" focus back to the caller + r_options = copy(options) + r_options.focus = "recall" + # strip comment containers from the escalation input (dup risk if captured, reader comments if not), + # ol/ul comment lists only if captured (#881). Not on a thread-forum, where the retry rescues the posts. + # Comments off: already pruned + esc_tree = ( + tree + if is_forum or not options.comments + else prune_unwanted_nodes(copy(tree), REMOVE_COMMENTS_AND_LISTS_XPATH if temp_comments else REMOVE_COMMENTS_XPATH) + ) + r_text = "" + try: + r_body, r_text = _recall_retry(esc_tree, r_options, url) + except Exception as err: # pragma: no cover + LOGGER.warning("recall retry failed: %s %s", err, url) + # justext reaches div-buried content the rule retry misses (gated: ungated regressed + # own-fallback). No region scoping of its own -> esc_tree is comment-pruned above + j_text = "" + if not options.fast: + try: + j_body, j_text = justext_rescue(copy(esc_tree), options) + except Exception as err: # pragma: no cover + LOGGER.warning("justext candidate failed: %s %s", err, url) + + # floor on the retry: a result below the pipeline's own minimum is noise (the retry's + # former internal baseline used to displace such outputs). cookie/consent banners justext + # could pick up are pruned from its input in basic_cleaning. An accepted candidate saw the + # full page, so its exclusions are deliberate -> drop the forum-post salvage. + len_text, r_len, j_len = len(temp_text), len(r_text), len(j_text) + if j_len > r_len and j_len > ESCALATION_JUSTEXT_RATIO * len_text: + postbody, temp_text, forum_posts = j_body, j_text, None + elif r_len >= options.min_extracted_size and r_len > ESCALATION_ACCEPT_RATIO * len_text: + postbody, temp_text, forum_posts = r_body, r_text, None + + if forum_posts is not None: + # a gate (escalation length, precision) blocked the cascade from restoring the posts: + # append the ones missing from the body + existing = "\n".join(filter(None, (_elem_text(el) for el in postbody))) + salvaged = [el for el in forum_posts if (t := _elem_text(el)) and t not in existing] + if salvaged: + LOGGER.debug("thread-forum salvage: %s captured posts appended to the body", len(salvaged)) + postbody.extend(salvaged) + temp_text = " ".join(postbody.itertext()).strip() + + return postbody, temp_text, commentsbody, temp_comments + + +def bare_extraction( + filecontent: HtmlInput, + url: str | None = None, + fast: bool = False, + no_fallback: bool = False, + favor_precision: bool = False, + favor_recall: bool = False, + include_comments: bool = True, + output_format: str = "python", + target_language: str | None = None, + include_tables: bool = True, + include_images: bool = False, + include_formatting: bool | None = None, + include_links: bool = False, + deduplicate: bool | LRUCache = False, + date_extraction_params: dict[str, Any] | None = None, + with_metadata: bool = False, + only_with_metadata: bool = False, + max_tree_size: int | None = None, + url_blacklist: set[str] | None = None, + author_blacklist: set[str] | None = None, + as_dict: bool = False, + prune_xpath: str | list[str] | None = None, + config: ConfigParser = DEFAULT_CONFIG, + options: Extractor | None = None, +) -> Document | dict[str, Any] | None: + """Internal function for text extraction returning bare Python variables. + + Args: + filecontent: HTML code as string. + url: URL of the webpage. + fast: Use faster heuristics and skip backup extraction. + no_fallback: Deprecated, use "fast" instead. + favor_precision: prefer less text but correct extraction. + favor_recall: prefer more text even when unsure. + include_comments: Extract comments along with the main text. + output_format: Define an output format, Python being the default + and the interest of this internal function. + Other values: "csv", "html", "json", "markdown", "txt", "xml", and "xmltei". + target_language: Define a language to discard invalid documents (ISO 639-1 format). + include_tables: Take into account information within the HTML element. + include_images: Take images into account. + include_formatting: Keep structural elements related to formatting + (kept in XML, rendered as markdown for text formats; ignored for JSON). + include_links: Keep links along with their targets. + deduplicate: Remove duplicate segments and documents. + Accepts an LRUCache instance as dedicated, scoped cache. + date_extraction_params: Provide extraction parameters to htmldate as dict(). + with_metadata: Extract metadata fields and add them to the output. + only_with_metadata: Only keep documents featuring all essential metadata + (date, title, url). + url_blacklist: Provide a blacklist of URLs as set() to filter out documents. + author_blacklist: Provide a blacklist of Author Names as set() to filter out authors. + as_dict: Deprecated, use the .as_dict() method instead. + prune_xpath: Provide an XPath expression to prune the tree before extraction. + can be str or list of str. + config: Directly provide a configparser configuration. + options: Directly provide a whole extractor configuration. + + Returns: + A Python dict() containing all the extracted information or None. + + Raises: + ValueError: Extraction problem. + + Note: + Low-level primitive: returns a Document with an unserialized .body tree; tei_validation only applies when serializing via extract(). + In the default balanced mode, a short extraction covering little of the page is automatically retried with recall settings. + """ + + # deprecations: stacklevel=3 → user → bare_extraction → _check_deprecation + fast = _check_deprecation( + fast, + no_fallback=no_fallback, + as_dict=as_dict, + max_tree_size=max_tree_size, + stacklevel=3, + ) + + # regroup extraction options + if not options or not isinstance(options, Extractor): + options = Extractor( + config=config, + output_format=output_format, + fast=fast, + precision=favor_precision, + recall=favor_recall, + comments=include_comments, + formatting=include_formatting, + links=include_links, + images=include_images, + tables=include_tables, + dedup=deduplicate, + lang=target_language, + url=url, + with_metadata=with_metadata, + only_with_metadata=only_with_metadata, + author_blacklist=author_blacklist, + url_blacklist=url_blacklist, + date_params=date_extraction_params, + ) + + try: + # load the HTML tree + tree = load_html(filecontent, options.max_file_size) + if tree is None: + LOGGER.error("empty HTML tree: %s", url) + raise ValueError + if tree is filecontent: + tree = copy(tree) + + # quick and dirty HTML lang check + if options.lang and (options.fast or not LANGID_FLAG): + if check_html_lang(tree, options.lang) is False: + LOGGER.error("wrong HTML meta language: %s", options.source) + raise ValueError + + # extract metadata if necessary + if options.with_metadata: + document = extract_metadata( + tree, + options.url, + options.date_params, + options.fast, + options.author_blacklist, + ) + + # cut short if extracted URL in blacklist + if document.url in options.url_blacklist: + LOGGER.warning("blacklisted URL: %s", document.url) + raise ValueError + + # cut short if core elements are missing + if options.only_with_metadata and not (document.date and document.title and document.url): + LOGGER.error("no metadata: %s", options.source) + raise ValueError + + else: + document = Document() + + # prune all xpath expressions that user specified + # no backup as this is unetre full control of the user + if prune_xpath is not None: + if isinstance(prune_xpath, str): + prune_xpath = [prune_xpath] + tree = prune_unwanted_nodes(tree, [XPath(x) for x in prune_xpath]) + + postbody, temp_text, commentsbody, temp_comments = trafilatura_sequence(tree, options, options.url or document.url) + len_text, len_comments = len(temp_text), len(temp_comments) + + # tree size sanity check + if options.max_tree_size: + # strip tags + if len(postbody) > options.max_tree_size: + LOGGER.debug("output tree too long: %s", len(postbody)) + strip_tags(postbody, "hi") + # still too long, raise an error + if len(postbody) > options.max_tree_size: + LOGGER.debug( + "output tree too long: %s, discarding %s", + len(postbody), + options.source, + ) + raise ValueError + # size checks + if len_text < options.min_output_size and len_comments < options.min_output_comm_size: + LOGGER.debug( + "text and comments not long enough: %s %s %s", + len_text, + len_comments, + options.source, + ) + raise ValueError + + # check duplicates at body level + if options.dedup and duplicate_test(postbody, options) is True: + LOGGER.debug("discarding duplicate document: %s", options.source) + raise ValueError + + # sanity check on language + if options.lang: + is_not_target_lang, document = language_filter(temp_text, temp_comments, options.lang, document) + if is_not_target_lang is True: + LOGGER.debug("wrong language: %s", options.source) + raise ValueError + + except (TypeError, ValueError): + LOGGER.warning("discarding data: %s", options.source) + return None + + # special case: python variables + if options.format == "python": + document.text = xmltotxt(postbody, options.formatting) + if options.comments: + document.comments = xmltotxt(commentsbody, options.formatting) + document.commentsbody = commentsbody + document.raw_text = document.text + else: + document.raw_text, document.commentsbody = temp_text, commentsbody + document.body = postbody + + return document if not as_dict else document.as_dict() + + +def extract( + filecontent: HtmlInput, + url: str | None = None, + record_id: str | None = None, + fast: bool = False, + no_fallback: bool = False, + favor_precision: bool = False, + favor_recall: bool = False, + include_comments: bool = True, + output_format: str = "txt", + tei_validation: bool = False, + target_language: str | None = None, + include_tables: bool = True, + include_images: bool = False, + include_formatting: bool | None = None, + include_links: bool = False, + deduplicate: bool | LRUCache = False, + date_extraction_params: dict[str, Any] | None = None, + with_metadata: bool = False, + only_with_metadata: bool = False, + max_tree_size: int | None = None, + url_blacklist: set[str] | None = None, + author_blacklist: set[str] | None = None, + settingsfile: str | None = None, + prune_xpath: str | list[str] | None = None, + config: ConfigParser = DEFAULT_CONFIG, + options: Extractor | None = None, +) -> str | None: + """Main function exposed by the package: + Wrapper for text extraction and conversion to chosen output format. + + Args: + filecontent: HTML code as string. + url: URL of the webpage. + record_id: Add an ID to the metadata. + fast: Use faster heuristics and skip backup extraction. + no_fallback: Deprecated, use "fast" instead. + favor_precision: prefer less text but correct extraction. + favor_recall: when unsure, prefer more text. + include_comments: Extract comments along with the main text. + output_format: Define an output format: + "csv", "html", "json", "markdown", "txt", "xml", and "xmltei". + tei_validation: Validate the XML-TEI output with respect to the TEI standard. + target_language: Define a language to discard invalid documents (ISO 639-1 format). + include_tables: Take into account information within the HTML
element. + include_images: Take images into account. + include_formatting: Keep structural elements related to formatting + (kept in XML, rendered as markdown for text formats; ignored for JSON). + include_links: Keep links along with their targets. + deduplicate: Remove duplicate segments and documents. + Accepts an LRUCache instance as dedicated, scoped cache. + date_extraction_params: Provide extraction parameters to htmldate as dict(). + with_metadata: Extract metadata fields and add them to the output. + only_with_metadata: Only keep documents featuring all essential metadata + (date, title, url). + url_blacklist: Provide a blacklist of URLs as set() to filter out documents. + author_blacklist: Provide a blacklist of Author Names as set() to filter out authors. + settingsfile: Use a configuration file to override the standard settings. + prune_xpath: Provide an XPath expression to prune the tree before extraction. + can be str or list of str. + config: Directly provide a configparser configuration. + options: Directly provide a whole extractor configuration. + + Returns: + A string in the desired format or None. + + """ + # must stay the first statement: a signature drift raises TypeError + document = _internal_extraction(**locals()) + return document.text if document is not None else None + + +def extract_with_metadata( + filecontent: HtmlInput, + url: str | None = None, + record_id: str | None = None, + fast: bool = False, + favor_precision: bool = False, + favor_recall: bool = False, + include_comments: bool = True, + output_format: str = "txt", + tei_validation: bool = False, + target_language: str | None = None, + include_tables: bool = True, + include_images: bool = False, + include_formatting: bool | None = None, + include_links: bool = False, + deduplicate: bool | LRUCache = False, + date_extraction_params: dict[str, Any] | None = None, + url_blacklist: set[str] | None = None, + author_blacklist: set[str] | None = None, + settingsfile: str | None = None, + prune_xpath: str | list[str] | None = None, + config: ConfigParser = DEFAULT_CONFIG, + options: Extractor | None = None, +) -> Document | None: + """Main function exposed by the package: + Wrapper for text extraction and conversion to chosen output format. + This method also returns document metadata. + + Args: + filecontent: HTML code as string. + url: URL of the webpage. + record_id: Add an ID to the metadata. + fast: Use faster heuristics and skip backup extraction. + favor_precision: prefer less text but correct extraction. + favor_recall: when unsure, prefer more text. + include_comments: Extract comments along with the main text. + output_format: Define an output format: + "csv", "html", "json", "markdown", "txt", "xml", and "xmltei". + tei_validation: Validate the XML-TEI output with respect to the TEI standard. + target_language: Define a language to discard invalid documents (ISO 639-1 format). + include_tables: Take into account information within the HTML
element. + include_images: Take images into account. + include_formatting: Keep structural elements related to formatting + (kept in XML, rendered as markdown for text formats; ignored for JSON). + include_links: Keep links along with their targets. + deduplicate: Remove duplicate segments and documents. + Accepts an LRUCache instance as dedicated, scoped cache. + date_extraction_params: Provide extraction parameters to htmldate as dict(). + url_blacklist: Provide a blacklist of URLs as set() to filter out documents. + author_blacklist: Provide a blacklist of Author Names as set() to filter out authors. + settingsfile: Use a configuration file to override the standard settings. + prune_xpath: Provide an XPath expression to prune the tree before extraction. + can be str or list of str. + config: Directly provide a configparser configuration. + options: Directly provide a whole extractor configuration. + + Returns: + Document metadata with content string in the desired format or None. + """ + return _internal_extraction(**locals(), with_metadata=True) + + +def _check_deprecation( + fast: bool = False, + *, + no_fallback: bool = False, + as_dict: bool = False, + max_tree_size: int | None = None, + stacklevel: int = 2, +) -> bool: + """Check deprecated params and return the effective "fast" flag.""" + if no_fallback: + warnings.warn( + '"no_fallback" will be removed, use "fast" instead', + DeprecationWarning, + stacklevel=stacklevel, + ) + if as_dict: + warnings.warn( + '"as_dict" will be removed, use the .as_dict() method instead', + DeprecationWarning, + stacklevel=stacklevel, + ) + if max_tree_size: + raise ValueError('"max_tree_size" will be removed, use settings.cfg instead') + return fast or no_fallback + + +def _internal_extraction( + filecontent: HtmlInput, + url: str | None = None, + record_id: str | None = None, + fast: bool = False, + no_fallback: bool = False, + favor_precision: bool = False, + favor_recall: bool = False, + include_comments: bool = True, + output_format: str = "txt", + tei_validation: bool = False, + target_language: str | None = None, + include_tables: bool = True, + include_images: bool = False, + include_formatting: bool | None = None, + include_links: bool = False, + deduplicate: bool | LRUCache = False, + date_extraction_params: dict[str, Any] | None = None, + with_metadata: bool = False, + only_with_metadata: bool = False, + max_tree_size: int | None = None, + url_blacklist: set[str] | None = None, + author_blacklist: set[str] | None = None, + settingsfile: str | None = None, + prune_xpath: str | list[str] | None = None, + config: ConfigParser = DEFAULT_CONFIG, + options: Extractor | None = None, +) -> Document | None: + """Internal method to do the extraction""" + # stacklevel=4 → user → extract → _internal_extraction → _check_deprecation + fast = _check_deprecation( + fast, + no_fallback=no_fallback, + as_dict=False, + max_tree_size=max_tree_size, + stacklevel=4, + ) + + # regroup extraction options + if not options or not isinstance(options, Extractor): + options = Extractor( + config=use_config(settingsfile) if settingsfile else config, + output_format=output_format, + fast=fast, + precision=favor_precision, + recall=favor_recall, + comments=include_comments, + formatting=include_formatting, + links=include_links, + images=include_images, + tables=include_tables, + dedup=deduplicate, + lang=target_language, + url=url, + with_metadata=with_metadata, + only_with_metadata=only_with_metadata, + tei_validation=tei_validation, + author_blacklist=author_blacklist, + url_blacklist=url_blacklist, + date_params=date_extraction_params, + ) + + # extraction + document = bare_extraction( + filecontent, + options=options, + as_dict=False, + prune_xpath=prune_xpath, + ) + + # post-processing + if not document or not isinstance(document, Document): + return None + + # TXT formats only output the ID and fingerprint in their metadata header + if options.format not in TXT_FORMATS or options.with_metadata: + # control output + if options.format == "python": + raise ValueError("'python' format only usable in bare_extraction() function") + # add record ID to metadata + document.id = record_id + # calculate fingerprint + if document.raw_text is not None: + document.fingerprint = content_fingerprint(str(document.title) + " " + str(document.raw_text)) + + document.text = determine_returnstring(document, options) + return document