Upstream adbar/trafilatura state of the touched files before 1377ec86cc

trafilatura/core.py

@@ -0,0 +1,728 @@
+# pylint:disable-msg=E0611,I1101
+"""
+Extraction configuration and processing functions.
+"""
+
+import json
+import logging
+import re
+import warnings
+from configparser import ConfigParser
+from copy import copy
+from typing import Any
+
+from lxml.etree import Element, XPath, _Element, strip_tags
+from lxml.html import HtmlElement
+
+# own
+from .baseline import baseline, html2txt
+from .deduplication import LRUCache, content_fingerprint, duplicate_test
+from .external import compare_extraction, justext_rescue
+from .htmlprocessing import (
+    build_html_output,
+    convert_tags,
+    prune_unwanted_nodes,
+    tree_cleaning,
+)
+from .main_extractor import _elem_text, extract_comments, extract_content
+from .metadata import Document, extract_metadata
+from .settings import DEFAULT_CONFIG, Extractor, use_config
+from .utils import (
+    LANGID_FLAG,
+    HtmlInput,
+    check_html_lang,
+    language_filter,
+    load_html,
+    normalize_unicode,
+)
+from .xml import build_json_output, control_xml_output, delete_element, keeps_empty, xmltocsv, xmltotxt
+from .xpaths import (
+    RAW_TREE_PRUNE_XPATH,
+    REMOVE_COMMENTS_AND_LISTS_XPATH,
+    REMOVE_COMMENTS_XPATH,
+)
+
+LOGGER = logging.getLogger(__name__)
+
+# recall escalation (see trafilatura_sequence, stage 4): retry a short balanced extraction
+# in recall mode. Calibrated as a set against the benchmark suite (own-bench/WMB/WCXB/AEB) —
+# don't tune one value in isolation, and re-run the full suite after any change.
+ESCALATION_MAX_LENGTH = 3000  # only consider extractions below this size
+ESCALATION_PAGE_SHARE = 0.2  # ... covering less than this share of the page text
+ESCALATION_ACCEPT_RATIO = 1.5  # accept the retry if it is this much longer
+# justext tends to over-include (whole pricing tables, unrelated sections) rather than stop at
+# page boundaries the way the rule-based retry does, so it needs a stricter bar than the
+# retry's own 1.5x: swept 1.5-10x, 2.0x is the best balance across the full suite.
+ESCALATION_JUSTEXT_RATIO = 2.0
+
+TXT_FORMATS = {"markdown", "txt"}
+_YAML_FIELDS = "title author url hostname description sitename date categories tags fingerprint id license".split()
+
+# Metadata is emitted as a YAML-style Markdown header; values such as a title
+# containing ": " (or a leading indicator, or a reserved word) otherwise produce
+# invalid YAML or get reinterpreted as a non-string. See GH #814.
+_YAML_RESERVED = frozenset({"true", "false", "yes", "no", "on", "off", "y", "n", "null", "none", "~"})
+
+
+def _yaml_scalar(value: str) -> str:
+    "Render a metadata string as a plain or double-quoted YAML-safe scalar."
+    if (
+        value
+        and value == value.strip()
+        and value[0].isalpha()
+        and ": " not in value
+        and " #" not in value
+        and not value.endswith(":")
+        and value.lower() not in _YAML_RESERVED
+        and all(ch >= " " and ch != "\x7f" for ch in value)
+    ):
+        return value
+    # a JSON string is always a valid, exactly round-tripping YAML double-quoted scalar
+    return json.dumps(value, ensure_ascii=False)
+
+
+def determine_returnstring(document: Document, options: Extractor) -> str:
+    """Convert XML tree to chosen format, clean the result and output it as a string"""
+    # XML (TEI) steps
+    if "xml" in options.format:
+        # last cleaning
+        for element in document.body.iter("*"):
+            if len(element) == 0 and not element.text and not element.tail and not keeps_empty(element):
+                delete_element(element, keep_tail=False)
+        # build output tree
+        returnstring = control_xml_output(document, options)
+    # CSV
+    elif options.format == "csv":
+        returnstring = xmltocsv(document, options.formatting)
+    # JSON
+    elif options.format == "json":
+        returnstring = build_json_output(document, options.with_metadata)
+    # HTML
+    elif options.format == "html":
+        returnstring = build_html_output(document, options.with_metadata)
+    # Markdown and TXT
+    else:
+        header = ""
+        if options.with_metadata:
+            # categories/tags are lists, their repr is a valid flow sequence
+            fields = "".join(
+                f"{attr}: {_yaml_scalar(value) if isinstance(value, str) else value}\n"
+                for attr in _YAML_FIELDS
+                if (value := getattr(document, attr))
+            )
+            header = f"---\n{fields}---\n"
+        returnstring = f"{header}{xmltotxt(document.body, options.formatting)}"
+        if document.commentsbody is not None:
+            returnstring = f"{returnstring}\n{xmltotxt(document.commentsbody, options.formatting)}".strip()
+    # normalize Unicode format (defaults to NFC)
+    return normalize_unicode(returnstring)
+
+
+# matches "@type": "DiscussionForumPosting" or an @type array containing it, anchored to the
+# key so it can't fire on the words appearing in ordinary prose (e.g. a description field)
+_DISCUSSION_FORUM_POSTING_RE = re.compile(
+    r'"@type"\s*:\s*"DiscussionForumPosting"|"@type"\s*:\s*\[[^\]]*"DiscussionForumPosting"'
+)
+
+
+def _forum_thread_page(tree: HtmlElement) -> bool:
+    """Detect a thread-forum page where posts live in the same containers
+    REMOVE_COMMENTS_XPATH would otherwise prune -- comments are content here, unlike on
+    a blog or article. Seeded by schema.org DiscussionForumPosting alone. Q&A forums
+    (StackExchange, schema.org QAPage) are deliberately not matched: their answers live
+    outside comment containers. Misses forums that don't emit DiscussionForumPosting
+    (e.g. old Reddit) -- an accepted gap. Meant as a reusable seed for a future page-type
+    router, not a one-off check.
+    """
+    return any(
+        script.text and _DISCUSSION_FORUM_POSTING_RE.search(script.text)
+        for script in tree.iterfind('.//script[@type="application/ld+json"]')
+    )
+
+
+def _prepare_tree(tree: HtmlElement, options: Extractor, url: str | None) -> tuple[HtmlElement, HtmlElement]:
+    "Clean and convert a raw tree, returning (converted, pre-conversion backup)."
+    cleaned = tree_cleaning(copy(tree), options)
+    backup = copy(cleaned)
+    cleaned = convert_tags(cleaned, options, url)
+    return cleaned, backup
+
+
+def _extract_and_compare(
+    cleaned_tree: HtmlElement, cleaned_tree_backup: HtmlElement, tree: HtmlElement, options: Extractor
+) -> tuple[_Element, str]:
+    "Cascade stages 1-2: main extractor, then the external comparison unless in fast mode."
+    postbody, temp_text = extract_content(cleaned_tree, options)
+    if not options.fast:
+        postbody, temp_text = compare_extraction(cleaned_tree_backup, copy(tree), postbody, temp_text, options)
+    return postbody, temp_text
+
+
+def _recall_retry(esc_tree: HtmlElement, r_options: Extractor, url: str | None) -> tuple[_Element, str]:
+    """Stage-4 retry: re-run cascade stages 1-2 in recall mode on the escalation input
+    (arrives comment-pruned, or intact on a thread-forum where posts are content).
+    Deliberately no comment capture, no baseline (it already ran on the full page; on a
+    comment-pruned tree it only yields an indistinguishable boilerplate dump), no escalation."""
+    return _extract_and_compare(*_prepare_tree(esc_tree, r_options, url), esc_tree, r_options)
+
+
+def trafilatura_sequence(
+    tree: HtmlElement,
+    options: Extractor,
+    url: str | None = None,
+) -> tuple[_Element, str, _Element, str]:
+    """Prepare the raw tree (cleaning, tag conversion, comment handling), then execute the
+    standard cascade of extractors used by Trafilatura, each stage only engaging if the
+    previous one under-delivered:
+    1. main extractor (includes wild-text recovery for short documents)
+    2. comparison with external extractors (readability/justext), skipped in fast mode
+    3. baseline rescue on the original, uncleaned tree
+    4. recall escalation, if the result still covers little of the page: stages 1-2 re-run
+       in recall mode (_recall_retry), plus a justext candidate tried alongside (a different
+       algorithm, not just stricter rules, so it reaches content the rule-based retry cannot)
+    Returns the body and comments elements with their text.
+
+    Internal helper: its signature and 4-tuple return are not a stable API — call
+    ``bare_extraction``/``extract`` instead.
+    """
+    is_forum = _forum_thread_page(tree)
+    # raw-tree prune so the external extractors inherit it too: readability would otherwise
+    # pick the longest appended article over the real one
+    tree = prune_unwanted_nodes(tree, RAW_TREE_PRUNE_XPATH)
+    # comments off: prune the raw tree so all stages inherit it, lists are still ol/ul here
+    if not options.comments and (options.focus == "precision" or not is_forum):
+        tree = prune_unwanted_nodes(tree, REMOVE_COMMENTS_AND_LISTS_XPATH)
+    cleaned_tree, cleaned_tree_backup = _prepare_tree(tree, options, url)
+
+    commentsbody, temp_comments = Element("body"), ""
+    forum_posts = None
+    if options.comments:
+        commentsbody, temp_comments, cleaned_tree = extract_comments(cleaned_tree, options)
+        if temp_comments and is_forum:
+            # thread-forum: the "comments" are the posts -> route into the body (backup predates
+            # capture); keep the capture aside, salvaged below if the cascade drops the posts
+            forum_posts = commentsbody
+            commentsbody, temp_comments = Element("body"), ""
+            cleaned_tree = convert_tags(copy(cleaned_tree_backup), options, url)
+    if options.focus == "precision" and not is_forum:
+        # NOT redundant with the raw-tree prune above: this runs POST-conversion, where lists are <list>
+        # and match the xpath's self::list (dl lists, disqus ids, and all lists if comments are on)
+        cleaned_tree = prune_unwanted_nodes(cleaned_tree, REMOVE_COMMENTS_XPATH)
+
+    postbody, temp_text = _extract_and_compare(cleaned_tree, cleaned_tree_backup, tree, options)
+
+    # 3. rescue: baseline on the original tree, accepted only if it adds text (#896)
+    if len(temp_text) < options.min_extracted_size and options.focus != "precision":
+        b_body, b_text, b_len = baseline(tree)  # baseline copies element inputs
+        LOGGER.debug("non-clean extracted length: %s (extraction)", b_len)
+        if b_len > len(temp_text):
+            postbody, temp_text = b_body, b_text
+            forum_posts = None  # the dump saw the whole page: missing posts are boilerplate, not lost
+
+    # 4. recall escalation: a short extraction covering little of the page suggests
+    # under-extraction (non-article layout) — retry in recall mode, keep if clearly bigger.
+    # NOTE: the page measure html2txt(tree) is coupled to BASIC_CLEAN_XPATH — see settings.py.
+    if (
+        options.focus == "balanced"
+        and 0 < len(temp_text) < ESCALATION_MAX_LENGTH
+        and len(temp_text) < ESCALATION_PAGE_SHARE * len(html2txt(tree))
+    ):
+        # a copy so a shared Extractor never leaks the "recall" focus back to the caller
+        r_options = copy(options)
+        r_options.focus = "recall"
+        # strip comment containers from the escalation input (dup risk if captured, reader comments if not),
+        # ol/ul comment lists only if captured (#881). Not on a thread-forum, where the retry rescues the posts.
+        # Comments off: already pruned
+        esc_tree = (
+            tree
+            if is_forum or not options.comments
+            else prune_unwanted_nodes(copy(tree), REMOVE_COMMENTS_AND_LISTS_XPATH if temp_comments else REMOVE_COMMENTS_XPATH)
+        )
+        r_text = ""
+        try:
+            r_body, r_text = _recall_retry(esc_tree, r_options, url)
+        except Exception as err:  # pragma: no cover
+            LOGGER.warning("recall retry failed: %s %s", err, url)
+        # justext reaches div-buried content the rule retry misses (gated: ungated regressed
+        # own-fallback). No region scoping of its own -> esc_tree is comment-pruned above
+        j_text = ""
+        if not options.fast:
+            try:
+                j_body, j_text = justext_rescue(copy(esc_tree), options)
+            except Exception as err:  # pragma: no cover
+                LOGGER.warning("justext candidate failed: %s %s", err, url)
+
+        # floor on the retry: a result below the pipeline's own minimum is noise (the retry's
+        # former internal baseline used to displace such outputs). cookie/consent banners justext
+        # could pick up are pruned from its input in basic_cleaning. An accepted candidate saw the
+        # full page, so its exclusions are deliberate -> drop the forum-post salvage.
+        len_text, r_len, j_len = len(temp_text), len(r_text), len(j_text)
+        if j_len > r_len and j_len > ESCALATION_JUSTEXT_RATIO * len_text:
+            postbody, temp_text, forum_posts = j_body, j_text, None
+        elif r_len >= options.min_extracted_size and r_len > ESCALATION_ACCEPT_RATIO * len_text:
+            postbody, temp_text, forum_posts = r_body, r_text, None
+
+    if forum_posts is not None:
+        # a gate (escalation length, precision) blocked the cascade from restoring the posts:
+        # append the ones missing from the body
+        existing = "\n".join(filter(None, (_elem_text(el) for el in postbody)))
+        salvaged = [el for el in forum_posts if (t := _elem_text(el)) and t not in existing]
+        if salvaged:
+            LOGGER.debug("thread-forum salvage: %s captured posts appended to the body", len(salvaged))
+            postbody.extend(salvaged)
+            temp_text = " ".join(postbody.itertext()).strip()
+
+    return postbody, temp_text, commentsbody, temp_comments
+
+
+def bare_extraction(
+    filecontent: HtmlInput,
+    url: str | None = None,
+    fast: bool = False,
+    no_fallback: bool = False,
+    favor_precision: bool = False,
+    favor_recall: bool = False,
+    include_comments: bool = True,
+    output_format: str = "python",
+    target_language: str | None = None,
+    include_tables: bool = True,
+    include_images: bool = False,
+    include_formatting: bool | None = None,
+    include_links: bool = False,
+    deduplicate: bool | LRUCache = False,
+    date_extraction_params: dict[str, Any] | None = None,
+    with_metadata: bool = False,
+    only_with_metadata: bool = False,
+    max_tree_size: int | None = None,
+    url_blacklist: set[str] | None = None,
+    author_blacklist: set[str] | None = None,
+    as_dict: bool = False,
+    prune_xpath: str | list[str] | None = None,
+    config: ConfigParser = DEFAULT_CONFIG,
+    options: Extractor | None = None,
+) -> Document | dict[str, Any] | None:
+    """Internal function for text extraction returning bare Python variables.
+
+    Args:
+        filecontent: HTML code as string.
+        url: URL of the webpage.
+        fast: Use faster heuristics and skip backup extraction.
+        no_fallback: Deprecated, use "fast" instead.
+        favor_precision: prefer less text but correct extraction.
+        favor_recall: prefer more text even when unsure.
+        include_comments: Extract comments along with the main text.
+        output_format: Define an output format, Python being the default
+            and the interest of this internal function.
+            Other values: "csv", "html", "json", "markdown", "txt", "xml", and "xmltei".
+        target_language: Define a language to discard invalid documents (ISO 639-1 format).
+        include_tables: Take into account information within the HTML <table> element.
+        include_images: Take images into account.
+        include_formatting: Keep structural elements related to formatting
+            (kept in XML, rendered as markdown for text formats; ignored for JSON).
+        include_links: Keep links along with their targets.
+        deduplicate: Remove duplicate segments and documents.
+            Accepts an LRUCache instance as dedicated, scoped cache.
+        date_extraction_params: Provide extraction parameters to htmldate as dict().
+        with_metadata: Extract metadata fields and add them to the output.
+        only_with_metadata: Only keep documents featuring all essential metadata
+            (date, title, url).
+        url_blacklist: Provide a blacklist of URLs as set() to filter out documents.
+        author_blacklist: Provide a blacklist of Author Names as set() to filter out authors.
+        as_dict: Deprecated, use the .as_dict() method instead.
+        prune_xpath: Provide an XPath expression to prune the tree before extraction.
+            can be str or list of str.
+        config: Directly provide a configparser configuration.
+        options: Directly provide a whole extractor configuration.
+
+    Returns:
+        A Python dict() containing all the extracted information or None.
+
+    Raises:
+        ValueError: Extraction problem.
+
+    Note:
+        Low-level primitive: returns a Document with an unserialized .body tree; tei_validation only applies when serializing via extract().
+        In the default balanced mode, a short extraction covering little of the page is automatically retried with recall settings.
+    """
+
+    # deprecations: stacklevel=3 → user → bare_extraction → _check_deprecation
+    fast = _check_deprecation(
+        fast,
+        no_fallback=no_fallback,
+        as_dict=as_dict,
+        max_tree_size=max_tree_size,
+        stacklevel=3,
+    )
+
+    # regroup extraction options
+    if not options or not isinstance(options, Extractor):
+        options = Extractor(
+            config=config,
+            output_format=output_format,
+            fast=fast,
+            precision=favor_precision,
+            recall=favor_recall,
+            comments=include_comments,
+            formatting=include_formatting,
+            links=include_links,
+            images=include_images,
+            tables=include_tables,
+            dedup=deduplicate,
+            lang=target_language,
+            url=url,
+            with_metadata=with_metadata,
+            only_with_metadata=only_with_metadata,
+            author_blacklist=author_blacklist,
+            url_blacklist=url_blacklist,
+            date_params=date_extraction_params,
+        )
+
+    try:
+        # load the HTML tree
+        tree = load_html(filecontent, options.max_file_size)
+        if tree is None:
+            LOGGER.error("empty HTML tree: %s", url)
+            raise ValueError
+        if tree is filecontent:
+            tree = copy(tree)
+
+        # quick and dirty HTML lang check
+        if options.lang and (options.fast or not LANGID_FLAG):
+            if check_html_lang(tree, options.lang) is False:
+                LOGGER.error("wrong HTML meta language: %s", options.source)
+                raise ValueError
+
+        # extract metadata if necessary
+        if options.with_metadata:
+            document = extract_metadata(
+                tree,
+                options.url,
+                options.date_params,
+                options.fast,
+                options.author_blacklist,
+            )
+
+            # cut short if extracted URL in blacklist
+            if document.url in options.url_blacklist:
+                LOGGER.warning("blacklisted URL: %s", document.url)
+                raise ValueError
+
+            # cut short if core elements are missing
+            if options.only_with_metadata and not (document.date and document.title and document.url):
+                LOGGER.error("no metadata: %s", options.source)
+                raise ValueError
+
+        else:
+            document = Document()
+
+        # prune all xpath expressions that user specified
+        # no backup as this is unetre full control of the user
+        if prune_xpath is not None:
+            if isinstance(prune_xpath, str):
+                prune_xpath = [prune_xpath]
+            tree = prune_unwanted_nodes(tree, [XPath(x) for x in prune_xpath])
+
+        postbody, temp_text, commentsbody, temp_comments = trafilatura_sequence(tree, options, options.url or document.url)
+        len_text, len_comments = len(temp_text), len(temp_comments)
+
+        # tree size sanity check
+        if options.max_tree_size:
+            # strip tags
+            if len(postbody) > options.max_tree_size:
+                LOGGER.debug("output tree too long: %s", len(postbody))
+                strip_tags(postbody, "hi")
+            # still too long, raise an error
+            if len(postbody) > options.max_tree_size:
+                LOGGER.debug(
+                    "output tree too long: %s, discarding %s",
+                    len(postbody),
+                    options.source,
+                )
+                raise ValueError
+        # size checks
+        if len_text < options.min_output_size and len_comments < options.min_output_comm_size:
+            LOGGER.debug(
+                "text and comments not long enough: %s %s %s",
+                len_text,
+                len_comments,
+                options.source,
+            )
+            raise ValueError
+
+        # check duplicates at body level
+        if options.dedup and duplicate_test(postbody, options) is True:
+            LOGGER.debug("discarding duplicate document: %s", options.source)
+            raise ValueError
+
+        # sanity check on language
+        if options.lang:
+            is_not_target_lang, document = language_filter(temp_text, temp_comments, options.lang, document)
+            if is_not_target_lang is True:
+                LOGGER.debug("wrong language: %s", options.source)
+                raise ValueError
+
+    except (TypeError, ValueError):
+        LOGGER.warning("discarding data: %s", options.source)
+        return None
+
+    # special case: python variables
+    if options.format == "python":
+        document.text = xmltotxt(postbody, options.formatting)
+        if options.comments:
+            document.comments = xmltotxt(commentsbody, options.formatting)
+            document.commentsbody = commentsbody
+        document.raw_text = document.text
+    else:
+        document.raw_text, document.commentsbody = temp_text, commentsbody
+    document.body = postbody
+
+    return document if not as_dict else document.as_dict()
+
+
+def extract(
+    filecontent: HtmlInput,
+    url: str | None = None,
+    record_id: str | None = None,
+    fast: bool = False,
+    no_fallback: bool = False,
+    favor_precision: bool = False,
+    favor_recall: bool = False,
+    include_comments: bool = True,
+    output_format: str = "txt",
+    tei_validation: bool = False,
+    target_language: str | None = None,
+    include_tables: bool = True,
+    include_images: bool = False,
+    include_formatting: bool | None = None,
+    include_links: bool = False,
+    deduplicate: bool | LRUCache = False,
+    date_extraction_params: dict[str, Any] | None = None,
+    with_metadata: bool = False,
+    only_with_metadata: bool = False,
+    max_tree_size: int | None = None,
+    url_blacklist: set[str] | None = None,
+    author_blacklist: set[str] | None = None,
+    settingsfile: str | None = None,
+    prune_xpath: str | list[str] | None = None,
+    config: ConfigParser = DEFAULT_CONFIG,
+    options: Extractor | None = None,
+) -> str | None:
+    """Main function exposed by the package:
+       Wrapper for text extraction and conversion to chosen output format.
+
+    Args:
+        filecontent: HTML code as string.
+        url: URL of the webpage.
+        record_id: Add an ID to the metadata.
+        fast: Use faster heuristics and skip backup extraction.
+        no_fallback: Deprecated, use "fast" instead.
+        favor_precision: prefer less text but correct extraction.
+        favor_recall: when unsure, prefer more text.
+        include_comments: Extract comments along with the main text.
+        output_format: Define an output format:
+            "csv", "html", "json", "markdown", "txt", "xml", and "xmltei".
+        tei_validation: Validate the XML-TEI output with respect to the TEI standard.
+        target_language: Define a language to discard invalid documents (ISO 639-1 format).
+        include_tables: Take into account information within the HTML <table> element.
+        include_images: Take images into account.
+        include_formatting: Keep structural elements related to formatting
+            (kept in XML, rendered as markdown for text formats; ignored for JSON).
+        include_links: Keep links along with their targets.
+        deduplicate: Remove duplicate segments and documents.
+            Accepts an LRUCache instance as dedicated, scoped cache.
+        date_extraction_params: Provide extraction parameters to htmldate as dict().
+        with_metadata: Extract metadata fields and add them to the output.
+        only_with_metadata: Only keep documents featuring all essential metadata
+            (date, title, url).
+        url_blacklist: Provide a blacklist of URLs as set() to filter out documents.
+        author_blacklist: Provide a blacklist of Author Names as set() to filter out authors.
+        settingsfile: Use a configuration file to override the standard settings.
+        prune_xpath: Provide an XPath expression to prune the tree before extraction.
+            can be str or list of str.
+        config: Directly provide a configparser configuration.
+        options: Directly provide a whole extractor configuration.
+
+    Returns:
+        A string in the desired format or None.
+
+    """
+    # must stay the first statement: a signature drift raises TypeError
+    document = _internal_extraction(**locals())
+    return document.text if document is not None else None
+
+
+def extract_with_metadata(
+    filecontent: HtmlInput,
+    url: str | None = None,
+    record_id: str | None = None,
+    fast: bool = False,
+    favor_precision: bool = False,
+    favor_recall: bool = False,
+    include_comments: bool = True,
+    output_format: str = "txt",
+    tei_validation: bool = False,
+    target_language: str | None = None,
+    include_tables: bool = True,
+    include_images: bool = False,
+    include_formatting: bool | None = None,
+    include_links: bool = False,
+    deduplicate: bool | LRUCache = False,
+    date_extraction_params: dict[str, Any] | None = None,
+    url_blacklist: set[str] | None = None,
+    author_blacklist: set[str] | None = None,
+    settingsfile: str | None = None,
+    prune_xpath: str | list[str] | None = None,
+    config: ConfigParser = DEFAULT_CONFIG,
+    options: Extractor | None = None,
+) -> Document | None:
+    """Main function exposed by the package:
+       Wrapper for text extraction and conversion to chosen output format.
+       This method also returns document metadata.
+
+    Args:
+        filecontent: HTML code as string.
+        url: URL of the webpage.
+        record_id: Add an ID to the metadata.
+        fast: Use faster heuristics and skip backup extraction.
+        favor_precision: prefer less text but correct extraction.
+        favor_recall: when unsure, prefer more text.
+        include_comments: Extract comments along with the main text.
+        output_format: Define an output format:
+            "csv", "html", "json", "markdown", "txt", "xml", and "xmltei".
+        tei_validation: Validate the XML-TEI output with respect to the TEI standard.
+        target_language: Define a language to discard invalid documents (ISO 639-1 format).
+        include_tables: Take into account information within the HTML <table> element.
+        include_images: Take images into account.
+        include_formatting: Keep structural elements related to formatting
+            (kept in XML, rendered as markdown for text formats; ignored for JSON).
+        include_links: Keep links along with their targets.
+        deduplicate: Remove duplicate segments and documents.
+            Accepts an LRUCache instance as dedicated, scoped cache.
+        date_extraction_params: Provide extraction parameters to htmldate as dict().
+        url_blacklist: Provide a blacklist of URLs as set() to filter out documents.
+        author_blacklist: Provide a blacklist of Author Names as set() to filter out authors.
+        settingsfile: Use a configuration file to override the standard settings.
+        prune_xpath: Provide an XPath expression to prune the tree before extraction.
+            can be str or list of str.
+        config: Directly provide a configparser configuration.
+        options: Directly provide a whole extractor configuration.
+
+    Returns:
+        Document metadata with content string in the desired format or None.
+    """
+    return _internal_extraction(**locals(), with_metadata=True)
+
+
+def _check_deprecation(
+    fast: bool = False,
+    *,
+    no_fallback: bool = False,
+    as_dict: bool = False,
+    max_tree_size: int | None = None,
+    stacklevel: int = 2,
+) -> bool:
+    """Check deprecated params and return the effective "fast" flag."""
+    if no_fallback:
+        warnings.warn(
+            '"no_fallback" will be removed, use "fast" instead',
+            DeprecationWarning,
+            stacklevel=stacklevel,
+        )
+    if as_dict:
+        warnings.warn(
+            '"as_dict" will be removed, use the .as_dict() method instead',
+            DeprecationWarning,
+            stacklevel=stacklevel,
+        )
+    if max_tree_size:
+        raise ValueError('"max_tree_size" will be removed, use settings.cfg instead')
+    return fast or no_fallback
+
+
+def _internal_extraction(
+    filecontent: HtmlInput,
+    url: str | None = None,
+    record_id: str | None = None,
+    fast: bool = False,
+    no_fallback: bool = False,
+    favor_precision: bool = False,
+    favor_recall: bool = False,
+    include_comments: bool = True,
+    output_format: str = "txt",
+    tei_validation: bool = False,
+    target_language: str | None = None,
+    include_tables: bool = True,
+    include_images: bool = False,
+    include_formatting: bool | None = None,
+    include_links: bool = False,
+    deduplicate: bool | LRUCache = False,
+    date_extraction_params: dict[str, Any] | None = None,
+    with_metadata: bool = False,
+    only_with_metadata: bool = False,
+    max_tree_size: int | None = None,
+    url_blacklist: set[str] | None = None,
+    author_blacklist: set[str] | None = None,
+    settingsfile: str | None = None,
+    prune_xpath: str | list[str] | None = None,
+    config: ConfigParser = DEFAULT_CONFIG,
+    options: Extractor | None = None,
+) -> Document | None:
+    """Internal method to do the extraction"""
+    # stacklevel=4 → user → extract → _internal_extraction → _check_deprecation
+    fast = _check_deprecation(
+        fast,
+        no_fallback=no_fallback,
+        as_dict=False,
+        max_tree_size=max_tree_size,
+        stacklevel=4,
+    )
+
+    # regroup extraction options
+    if not options or not isinstance(options, Extractor):
+        options = Extractor(
+            config=use_config(settingsfile) if settingsfile else config,
+            output_format=output_format,
+            fast=fast,
+            precision=favor_precision,
+            recall=favor_recall,
+            comments=include_comments,
+            formatting=include_formatting,
+            links=include_links,
+            images=include_images,
+            tables=include_tables,
+            dedup=deduplicate,
+            lang=target_language,
+            url=url,
+            with_metadata=with_metadata,
+            only_with_metadata=only_with_metadata,
+            tei_validation=tei_validation,
+            author_blacklist=author_blacklist,
+            url_blacklist=url_blacklist,
+            date_params=date_extraction_params,
+        )
+
+    # extraction
+    document = bare_extraction(
+        filecontent,
+        options=options,
+        as_dict=False,
+        prune_xpath=prune_xpath,
+    )
+
+    # post-processing
+    if not document or not isinstance(document, Document):
+        return None
+
+    # TXT formats only output the ID and fingerprint in their metadata header
+    if options.format not in TXT_FORMATS or options.with_metadata:
+        # control output
+        if options.format == "python":
+            raise ValueError("'python' format only usable in bare_extraction() function")
+        # add record ID to metadata
+        document.id = record_id
+        # calculate fingerprint
+        if document.raw_text is not None:
+            document.fingerprint = content_fingerprint(str(document.title) + " " + str(document.raw_text))
+
+    document.text = determine_returnstring(document, options)
+    return document