tests: cover listing pages across settings, precision and single-article pages (#774)

tests/unit_tests.py

@@ -2772,7 +2772,7 @@ def test_no_duplicate_content():
         assert out.count("Second synthetic paragraph") == 1
 
 
-def _listing_page(entries):
+def _listing_page(entries, container="main"):
     "Blog-index markup: sibling <article> teasers, each a link-only heading plus its lead paragraph."
     # each teaser clears min_extracted_size on its own, so only the page-share
     # test flags the layout as under-extracted and escalates to recall
@@ -2782,12 +2782,12 @@ def _listing_page(entries):
         f'<div class="entry-summary"><p>{text}</p></div></article>'
         for i, (title, text) in enumerate(entries)
     )
-    return f'<html><body><div id="wrapper"><main>{articles}</main></div></body></html>'
+    tag = container.split()[0]
+    return f'<html><body><div id="wrapper"><{container}>{articles}</{tag}></div></body></html>'
 
 
-def test_listing_page_keeps_every_entry_heading():
+def _listing_entries():
-    "regression #774: an index page must not collapse to its first entry, losing every other heading."
+    return [
-    entries = [
         (
             f"Entry number {i} about a self contained subject",
             (
@@ -2799,9 +2799,80 @@ def test_listing_page_keeps_every_entry_heading():
         )
         for i in range(8)
     ]
-    result = extract(_listing_page(entries), output_format="markdown", config=use_config()) or ""
+
-    for title, _ in entries:
+
+def _without_listing_expression():
+    "The extractor as it was before #774: the listing expression never matches."
+    return patch("trafilatura.main_extractor.LISTING_BODY_XPATH", [etree.XPath("self::listing-disabled")])
+
+
+@pytest.mark.parametrize(
+    "settings",
+    [{}, {"fast": True}, {"favor_recall": True}, {"favor_recall": True, "fast": True}],
+)
+@pytest.mark.parametrize("container", ["main", 'div class="posts"'])
+def test_listing_page_keeps_every_entry_heading(settings, container):
+    "regression #774: an index page must not collapse to its first entry, losing every other heading."
+    entries = _listing_entries()
+    result = extract(_listing_page(entries, container), output_format="markdown", config=use_config(), **settings) or ""
+    for title, text in entries:
         assert f"## {title}" in result
+        assert text in result
+
+
+@pytest.mark.parametrize("container", ["main", 'div class="posts"'])
+def test_listing_page_precision_unchanged(container):
+    "favor_precision never takes the listing container (#774), its output stays what it was."
+    doc = _listing_page(_listing_entries(), container)
+    with _without_listing_expression():
+        expected = extract(doc, output_format="markdown", favor_precision=True, config=use_config())
+    result = extract(doc, output_format="markdown", favor_precision=True, config=use_config())
+    assert result
+    assert result == expected
+
+
+@pytest.mark.parametrize("settings", [{}, {"favor_recall": True}, {"favor_precision": True}])
+def test_single_article_page_unaffected_by_listing_expression(settings):
+    "a page with one <article> has no sibling entries, so the listing expression must not change anything."
+    body = "".join(
+        f"<h2>Section {i} of the story</h2><p>Body paragraph {i} runs long enough to carry the page on "
+        "its own, with several clauses and a couple of sentences so the article clearly outweighs "
+        "anything beside it.</p>"
+        for i in range(5)
+    )
+    doc = (
+        f"<html><body><main><article><h1>The main article</h1>{body}</article>"
+        "<div><p>The author writes about gardens, soil and seasons, and has kept "
+        "a small allotment on the edge of town for more than twenty years now.</p></div>"
+        "</main></body></html>"
+    )
+    with _without_listing_expression():
+        expected = extract(doc, output_format="markdown", config=use_config(), **settings)
+    result = extract(doc, output_format="markdown", config=use_config(), **settings)
+    assert result == expected
+    for i in range(5):
+        assert f"## Section {i} of the story" in result
+    assert "allotment" not in result
+
+
+def test_listing_container_text_share():
+    "the listing expression only applies when the sibling-article container carries most of the page text"
+    from trafilatura.main_extractor import _holds_most_of_the_text
+    from trafilatura.xpaths import LISTING_BODY_XPATH
+
+    listing_expr = LISTING_BODY_XPATH[0]
+    listing = html.fromstring(_listing_page(_listing_entries()))
+    assert _holds_most_of_the_text(listing, listing_expr) is True
+
+    body = "<p>" + "A long body sentence that dominates the page. " * 20 + "</p>"
+    strip = _article_with_related_posts(body, [(f"Story {i}", f"Teaser {i}.") for i in range(3)])
+    assert _holds_most_of_the_text(html.fromstring(strip), listing_expr) is False
+
+    single = html.fromstring(f"<html><body><main><article>{body}</article></main></body></html>")
+    assert _holds_most_of_the_text(single, listing_expr) is False
+
+    empty = html.fromstring("<html><body><main><article></article><article></article></main></body></html>")
+    assert _holds_most_of_the_text(empty, listing_expr) is False
 
 
 def _article_with_related_posts(body, related):