diff --git a/tests/unit_tests.py b/tests/unit_tests.py
index 411ca1e..89179fd 100644
--- a/tests/unit_tests.py
+++ b/tests/unit_tests.py
@@ -2772,7 +2772,7 @@ def test_no_duplicate_content():
assert out.count("Second synthetic paragraph") == 1
-def _listing_page(entries):
+def _listing_page(entries, container="main"):
"Blog-index markup: sibling teasers, each a link-only heading plus its lead paragraph."
# each teaser clears min_extracted_size on its own, so only the page-share
# test flags the layout as under-extracted and escalates to recall
@@ -2782,12 +2782,12 @@ def _listing_page(entries):
f''
for i, (title, text) in enumerate(entries)
)
- return f'
{articles}
'
+ tag = container.split()[0]
+ return f'<{container}>{articles}{tag}>
'
-def test_listing_page_keeps_every_entry_heading():
- "regression #774: an index page must not collapse to its first entry, losing every other heading."
- entries = [
+def _listing_entries():
+ return [
(
f"Entry number {i} about a self contained subject",
(
@@ -2799,9 +2799,80 @@ def test_listing_page_keeps_every_entry_heading():
)
for i in range(8)
]
- result = extract(_listing_page(entries), output_format="markdown", config=use_config()) or ""
- for title, _ in entries:
+
+
+def _without_listing_expression():
+ "The extractor as it was before #774: the listing expression never matches."
+ return patch("trafilatura.main_extractor.LISTING_BODY_XPATH", [etree.XPath("self::listing-disabled")])
+
+
+@pytest.mark.parametrize(
+ "settings",
+ [{}, {"fast": True}, {"favor_recall": True}, {"favor_recall": True, "fast": True}],
+)
+@pytest.mark.parametrize("container", ["main", 'div class="posts"'])
+def test_listing_page_keeps_every_entry_heading(settings, container):
+ "regression #774: an index page must not collapse to its first entry, losing every other heading."
+ entries = _listing_entries()
+ result = extract(_listing_page(entries, container), output_format="markdown", config=use_config(), **settings) or ""
+ for title, text in entries:
assert f"## {title}" in result
+ assert text in result
+
+
+@pytest.mark.parametrize("container", ["main", 'div class="posts"'])
+def test_listing_page_precision_unchanged(container):
+ "favor_precision never takes the listing container (#774), its output stays what it was."
+ doc = _listing_page(_listing_entries(), container)
+ with _without_listing_expression():
+ expected = extract(doc, output_format="markdown", favor_precision=True, config=use_config())
+ result = extract(doc, output_format="markdown", favor_precision=True, config=use_config())
+ assert result
+ assert result == expected
+
+
+@pytest.mark.parametrize("settings", [{}, {"favor_recall": True}, {"favor_precision": True}])
+def test_single_article_page_unaffected_by_listing_expression(settings):
+ "a page with one has no sibling entries, so the listing expression must not change anything."
+ body = "".join(
+ f"Section {i} of the story
Body paragraph {i} runs long enough to carry the page on "
+ "its own, with several clauses and a couple of sentences so the article clearly outweighs "
+ "anything beside it.
"
+ for i in range(5)
+ )
+ doc = (
+ f"The main article
{body}"
+ "The author writes about gardens, soil and seasons, and has kept "
+ "a small allotment on the edge of town for more than twenty years now.
"
+ ""
+ )
+ with _without_listing_expression():
+ expected = extract(doc, output_format="markdown", config=use_config(), **settings)
+ result = extract(doc, output_format="markdown", config=use_config(), **settings)
+ assert result == expected
+ for i in range(5):
+ assert f"## Section {i} of the story" in result
+ assert "allotment" not in result
+
+
+def test_listing_container_text_share():
+ "the listing expression only applies when the sibling-article container carries most of the page text"
+ from trafilatura.main_extractor import _holds_most_of_the_text
+ from trafilatura.xpaths import LISTING_BODY_XPATH
+
+ listing_expr = LISTING_BODY_XPATH[0]
+ listing = html.fromstring(_listing_page(_listing_entries()))
+ assert _holds_most_of_the_text(listing, listing_expr) is True
+
+ body = "" + "A long body sentence that dominates the page. " * 20 + "
"
+ strip = _article_with_related_posts(body, [(f"Story {i}", f"Teaser {i}.") for i in range(3)])
+ assert _holds_most_of_the_text(html.fromstring(strip), listing_expr) is False
+
+ single = html.fromstring(f"{body}")
+ assert _holds_most_of_the_text(single, listing_expr) is False
+
+ empty = html.fromstring("")
+ assert _holds_most_of_the_text(empty, listing_expr) is False
def _article_with_related_posts(body, related):