# pylint:disable-msg=I1101,W1401
"""
Unit tests for the trafilatura library.
"""
import json
import logging
import sys
import time
from copy import copy
from functools import partial
from os import path
from unittest.mock import patch
import pytest
from lxml import etree, html
try:
from cchardet import detect
except ImportError:
from charset_normalizer import detect
import trafilatura.htmlprocessing
import trafilatura.xpaths as xp
from trafilatura import bare_extraction, baseline, core, external, extract, extract_with_metadata, xml
from trafilatura import main_extractor as me
from trafilatura.baseline import html2txt
from trafilatura.deduplication import LRU_TEST, LRUCache
from trafilatura.external import _prefer_readability, sanitize_tree, try_justext, try_readability
from trafilatura.htmlprocessing import convert_tags, tree_cleaning
from trafilatura.main_extractor import (
_span,
extract_content,
handle_formatting,
handle_image,
handle_lists,
handle_paragraphs,
handle_quotes,
handle_table,
handle_textelem,
handle_titles,
prune_unwanted_sections,
recover_wild_text,
)
from trafilatura.meta import reset_caches
from trafilatura.metadata import Document
from trafilatura.readability_lxml import Document as ReadabilityDocument
from trafilatura.readability_lxml import is_probably_readerable
from trafilatura.settings import MANUALLY_CLEANED, TAG_CATALOG, use_config
from trafilatura.utils import (
LANGID_FLAG,
detect_encoding,
is_dubious_html,
is_image_file,
language_classifier,
line_processing,
load_html,
normalize_unicode,
repair_faulty_html,
return_printables_and_spaces,
sanitize,
textfilter,
trim,
)
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
TEST_DIR = path.abspath(path.dirname(__file__))
RESOURCES_DIR = path.join(TEST_DIR, "resources")
SAMPLE_META = Document()
ZERO_CONFIG = use_config() # fresh copy: must not mutate the package-global DEFAULT_CONFIG
ZERO_CONFIG["DEFAULT"]["MIN_OUTPUT_SIZE"] = "0"
ZERO_CONFIG["DEFAULT"]["MIN_EXTRACTED_SIZE"] = "0"
NEW_CONFIG = use_config(filename=path.join(RESOURCES_DIR, "newsettings.cfg"))
_INTRO = "enough intro text here for extraction"
def _extract_doc(body, *, intro=True, **kwargs):
"Wrap body in an article doc and extract; config defaults to ZERO_CONFIG (min-sizes 0)."
kwargs.setdefault("config", ZERO_CONFIG)
inner = f"{_INTRO}
{body}" if intro else body
return extract(f"{inner}", **kwargs)
def _table_md(table, **kwargs):
return _extract_doc(table, output_format="markdown", include_tables=True, **kwargs)
def _table_txt(table):
return _extract_doc(table, intro=False, output_format="txt", fast=True, include_tables=True)
def _md_inline(body, **kwargs):
return _extract_doc(body, output_format="markdown", include_formatting=True, **kwargs) or ""
def _md(body, include_formatting=True):
"Serialize an XML snippet to markdown (or plain text), stripped."
return xml.xmltotxt(etree.fromstring(body), include_formatting=include_formatting).strip()
def _xpath_hits(xpaths, attrs, tag="div"):
"Whether any of the expressions selects a wrapping a paragraph."
root = etree.fromstring(f"<{tag} {attrs}>content
")
return any(len(x(root)) > 0 for x in xpaths)
def _handle_table(table, recall=False):
return handle_table(html.fromstring(table), TAG_CATALOG, core.Extractor(recall=recall))
MOCK_PAGES = {
"http://exotic_tags": "exotic_tags.html",
}
@pytest.fixture
def options():
"A fresh Extractor per test (never a shared, mutable module global)."
return core.Extractor()
def load_mock_page(url, xml_flag=False, langcheck=None, tei_output=False):
"""load mock page from samples"""
try:
with open(path.join(TEST_DIR, "resources", MOCK_PAGES[url]), encoding="utf-8") as inputf:
htmlstring = inputf.read()
# encoding/windows fix for the tests
except UnicodeDecodeError:
# read as binary
with open(path.join(TEST_DIR, "resources", MOCK_PAGES[url]), "rb") as inputf:
htmlbinary = inputf.read()
guessed_encoding = detect(htmlbinary)["encoding"]
if guessed_encoding is not None:
try:
htmlstring = htmlbinary.decode(guessed_encoding)
except UnicodeDecodeError:
htmlstring = htmlbinary
else:
print("Encoding error")
if xml_flag:
output_format = "xml"
elif tei_output:
output_format = "xmltei"
else:
output_format = "txt"
return extract(htmlstring, url, record_id="0000", output_format=output_format, target_language=langcheck)
def test_trim():
"""test string trimming"""
assert trim(" Test ") == "Test"
assert trim("\t\tTest Test\r\n") == "Test Test"
my_elem = etree.Element("body")
my_elem.text = "Test Text"
assert textfilter(my_elem) is False
# my_elem.text = 'Tags: Arbeit, Urlaub'
my_elem.text = "Instagram"
assert textfilter(my_elem) is True
my_elem.text = "\t\t"
assert textfilter(my_elem) is True
# sanitize logic
assert sanitize(None) is None
# non-breaking spaces
assert sanitize("Test Text") == "Test Text"
# clear cache
# reset caches: examine_date_elements used above
old_values = trim.cache_info()
reset_caches()
assert trim.cache_info() != old_values
def test_reset_caches():
"reset_caches() must empty every trafilatura-owned cache, not just trim."
caches = (trim, line_processing, return_printables_and_spaces)
for fn in caches:
fn("x")
LRU_TEST.put("k", 1)
assert all(fn.cache_info().currsize for fn in caches)
assert LRU_TEST.cache
reset_caches()
assert not any(fn.cache_info().currsize for fn in caches)
assert not LRU_TEST.cache
def test_input(options):
"""test if loaded strings/trees are handled properly"""
teststring = "高山云雾出好茶".encode()
assert detect_encoding(teststring) == ["utf-8"]
teststring = "高山云雾出好茶".encode("gb18030")
assert "gb18030" in detect_encoding(teststring)
assert "gb18030" in detect_encoding(teststring * 1000)
assert is_dubious_html("This is a string.") is True
htmlstring = "\n"
beginning = htmlstring[:50].lower()
assert repair_faulty_html(htmlstring, beginning) == "\n"
htmlstring = "\n"
beginning = htmlstring[:50].lower()
assert repair_faulty_html(htmlstring, beginning) == htmlstring
htmlstring = "\n"
beginning = htmlstring[:50].lower()
assert repair_faulty_html(htmlstring, beginning) == "\n"
htmlstring = '\n \n