From 7568e2a786ac8f06e79e7145d05fe2a3872abc96 Mon Sep 17 00:00:00 2001
From: upstream
Date: Fri, 11 Sep 2026 23:09:08 +0200
Subject: [PATCH] Upstream adbar/trafilatura state of the touched files before
49484caaab
---
tests/unit_tests.py | 4387 +++++++++++++++++++++++++++++++++
trafilatura/main_extractor.py | 792 ++++++
trafilatura/xpaths.py | 482 ++++
3 files changed, 5661 insertions(+)
create mode 100644 tests/unit_tests.py
create mode 100644 trafilatura/main_extractor.py
create mode 100644 trafilatura/xpaths.py
diff --git a/tests/unit_tests.py b/tests/unit_tests.py
new file mode 100644
index 0000000..977d4f5
--- /dev/null
+++ b/tests/unit_tests.py
@@ -0,0 +1,4387 @@
+# pylint:disable-msg=I1101,W1401
+"""
+Unit tests for the trafilatura library.
+"""
+
+import json
+import logging
+import sys
+import time
+from copy import copy
+from functools import partial
+from os import path
+from unittest.mock import patch
+
+import pytest
+from lxml import etree, html
+
+try:
+ from cchardet import detect
+except ImportError:
+ from charset_normalizer import detect
+
+import trafilatura.htmlprocessing
+import trafilatura.xpaths as xp
+from trafilatura import bare_extraction, baseline, core, external, extract, extract_with_metadata, xml
+from trafilatura import main_extractor as me
+from trafilatura.baseline import html2txt
+from trafilatura.deduplication import LRU_TEST, LRUCache
+from trafilatura.external import _prefer_readability, sanitize_tree, try_justext, try_readability
+from trafilatura.htmlprocessing import convert_tags, tree_cleaning
+from trafilatura.main_extractor import (
+ _span,
+ extract_content,
+ handle_formatting,
+ handle_image,
+ handle_lists,
+ handle_paragraphs,
+ handle_quotes,
+ handle_table,
+ handle_textelem,
+ handle_titles,
+ prune_unwanted_sections,
+ recover_wild_text,
+)
+from trafilatura.meta import reset_caches
+from trafilatura.metadata import Document
+from trafilatura.readability_lxml import Document as ReadabilityDocument
+from trafilatura.readability_lxml import is_probably_readerable
+from trafilatura.settings import MANUALLY_CLEANED, TAG_CATALOG, use_config
+from trafilatura.utils import (
+ LANGID_FLAG,
+ detect_encoding,
+ is_dubious_html,
+ is_image_file,
+ language_classifier,
+ line_processing,
+ load_html,
+ normalize_unicode,
+ repair_faulty_html,
+ return_printables_and_spaces,
+ sanitize,
+ textfilter,
+ trim,
+)
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+
+TEST_DIR = path.abspath(path.dirname(__file__))
+RESOURCES_DIR = path.join(TEST_DIR, "resources")
+SAMPLE_META = Document()
+
+ZERO_CONFIG = use_config() # fresh copy: must not mutate the package-global DEFAULT_CONFIG
+ZERO_CONFIG["DEFAULT"]["MIN_OUTPUT_SIZE"] = "0"
+ZERO_CONFIG["DEFAULT"]["MIN_EXTRACTED_SIZE"] = "0"
+
+NEW_CONFIG = use_config(filename=path.join(RESOURCES_DIR, "newsettings.cfg"))
+
+
+_INTRO = "enough intro text here for extraction"
+
+
+def _extract_doc(body, *, intro=True, **kwargs):
+ "Wrap body in an article doc and extract; config defaults to ZERO_CONFIG (min-sizes 0)."
+ kwargs.setdefault("config", ZERO_CONFIG)
+ inner = f"{_INTRO}
{body}" if intro else body
+ return extract(f"{inner}", **kwargs)
+
+
+def _table_md(table, **kwargs):
+ return _extract_doc(table, output_format="markdown", include_tables=True, **kwargs)
+
+
+def _table_txt(table):
+ return _extract_doc(table, intro=False, output_format="txt", fast=True, include_tables=True)
+
+
+def _md_inline(body, **kwargs):
+ return _extract_doc(body, output_format="markdown", include_formatting=True, **kwargs) or ""
+
+
+def _md(body, include_formatting=True):
+ "Serialize an XML snippet to markdown (or plain text), stripped."
+ return xml.xmltotxt(etree.fromstring(body), include_formatting=include_formatting).strip()
+
+
+def _xpath_hits(xpaths, attrs, tag="div"):
+ "Whether any of the expressions selects a wrapping a paragraph."
+ root = etree.fromstring(f"<{tag} {attrs}>content
")
+ return any(len(x(root)) > 0 for x in xpaths)
+
+
+def _handle_table(table, recall=False):
+ return handle_table(html.fromstring(table), TAG_CATALOG, core.Extractor(recall=recall))
+
+
+MOCK_PAGES = {
+ "http://exotic_tags": "exotic_tags.html",
+}
+
+
+@pytest.fixture
+def options():
+ "A fresh Extractor per test (never a shared, mutable module global)."
+ return core.Extractor()
+
+
+def load_mock_page(url, xml_flag=False, langcheck=None, tei_output=False):
+ """load mock page from samples"""
+ try:
+ with open(path.join(TEST_DIR, "resources", MOCK_PAGES[url]), encoding="utf-8") as inputf:
+ htmlstring = inputf.read()
+ # encoding/windows fix for the tests
+ except UnicodeDecodeError:
+ # read as binary
+ with open(path.join(TEST_DIR, "resources", MOCK_PAGES[url]), "rb") as inputf:
+ htmlbinary = inputf.read()
+ guessed_encoding = detect(htmlbinary)["encoding"]
+ if guessed_encoding is not None:
+ try:
+ htmlstring = htmlbinary.decode(guessed_encoding)
+ except UnicodeDecodeError:
+ htmlstring = htmlbinary
+ else:
+ print("Encoding error")
+ if xml_flag:
+ output_format = "xml"
+ elif tei_output:
+ output_format = "xmltei"
+ else:
+ output_format = "txt"
+ return extract(htmlstring, url, record_id="0000", output_format=output_format, target_language=langcheck)
+
+
+def test_trim():
+ """test string trimming"""
+ assert trim(" Test ") == "Test"
+ assert trim("\t\tTest Test\r\n") == "Test Test"
+ my_elem = etree.Element("body")
+ my_elem.text = "Test Text"
+ assert textfilter(my_elem) is False
+ # my_elem.text = 'Tags: Arbeit, Urlaub'
+ my_elem.text = "Instagram"
+ assert textfilter(my_elem) is True
+ my_elem.text = "\t\t"
+ assert textfilter(my_elem) is True
+ # sanitize logic
+ assert sanitize(None) is None
+ # non-breaking spaces
+ assert sanitize("Test Text") == "Test Text"
+ # clear cache
+ # reset caches: examine_date_elements used above
+ old_values = trim.cache_info()
+ reset_caches()
+ assert trim.cache_info() != old_values
+
+
+def test_reset_caches():
+ "reset_caches() must empty every trafilatura-owned cache, not just trim."
+ caches = (trim, line_processing, return_printables_and_spaces)
+ for fn in caches:
+ fn("x")
+ LRU_TEST.put("k", 1)
+ assert all(fn.cache_info().currsize for fn in caches)
+ assert LRU_TEST.cache
+ reset_caches()
+ assert not any(fn.cache_info().currsize for fn in caches)
+ assert not LRU_TEST.cache
+
+
+def test_input(options):
+ """test if loaded strings/trees are handled properly"""
+ teststring = "高山云雾出好茶".encode()
+ assert detect_encoding(teststring) == ["utf-8"]
+ teststring = "高山云雾出好茶".encode("gb18030")
+ assert "gb18030" in detect_encoding(teststring)
+ assert "gb18030" in detect_encoding(teststring * 1000)
+
+ assert is_dubious_html("This is a string.") is True
+
+ htmlstring = "\n"
+ beginning = htmlstring[:50].lower()
+ assert repair_faulty_html(htmlstring, beginning) == "\n"
+
+ htmlstring = "\n"
+ beginning = htmlstring[:50].lower()
+ assert repair_faulty_html(htmlstring, beginning) == htmlstring
+
+ htmlstring = "\n"
+ beginning = htmlstring[:50].lower()
+ assert repair_faulty_html(htmlstring, beginning) == "\n"
+
+ htmlstring = '\n\n \n