Use the listing expression only when its container holds most of the page text

tests/unit_tests.py

@@ -2779,7 +2779,7 @@ def _listing_page(entries):
     articles = "".join(
         f'<article><header class="entry-header"><h2 class="entry-title">'
         f'<a href="/{i}">{title}</a></h2></header>'
-        f"<div class=\"entry-summary\"><p>{text}</p></div></article>"
+        f'<div class="entry-summary"><p>{text}</p></div></article>'
         for i, (title, text) in enumerate(entries)
     )
     return f'<html><body><div id="wrapper"><main>{articles}</main></div></body></html>'
@@ -2790,10 +2790,12 @@ def test_listing_page_keeps_every_entry_heading():
     entries = [
         (
             f"Entry number {i} about a self contained subject",
+            (
                 f"Teaser {i} carries enough prose to pass the extractor thresholds on its own, running "
                 f"over several clauses and a couple of sentences. It reads as a lead paragraph would, "
                 f"so nothing about this entry looks like boilerplate to the extractor, and the index "
-            f"page as a whole reads as a real body of text rather than a stub.",
+                f"page as a whole reads as a real body of text rather than a stub."
+            ),
         )
         for i in range(8)
     ]
@@ -2802,6 +2804,38 @@ def test_listing_page_keeps_every_entry_heading():
         assert f"## {title}" in result
 
 
+def _article_with_related_posts(body, related):
+    "Article markup carrying a teaser strip: sibling <article> elements beside the real body."
+    teasers = "".join(
+        f'<article><header><h2 class="entry-title"><a href="/r{i}">{title}</a></h2></header><p>{text}</p></article>'
+        for i, (title, text) in enumerate(related)
+    )
+    return (
+        f'<html><body><div id="wrapper"><main>'
+        f"<article><h1>The main article</h1>{body}</article>"
+        f'<div class="related-posts"><h2>Recommended stories</h2>{teasers}</div>'
+        f"</main></div></body></html>"
+    )
+
+
+def test_related_posts_strip_does_not_replace_the_article_body():
+    "a teaser strip has the same shape as a listing, so recall must still return the body"
+    body = "".join(
+        f"<p>Body paragraph {i} runs long enough to carry the page on its own, with several "
+        f"clauses and a couple of sentences so the article clearly outweighs anything beside "
+        f"it. Nothing here looks like a teaser or like boilerplate to the extractor.</p>"
+        for i in range(6)
+    )
+    related = [(f"Recommended story {i}", f"Teaser {i}.") for i in range(3)]
+    html_doc = _article_with_related_posts(body, related)
+
+    result = extract(html_doc, output_format="markdown", favor_recall=True, config=use_config()) or ""
+
+    assert "Body paragraph 0" in result
+    assert "Body paragraph 5" in result
+    assert "Recommended story 0" not in result
+
+
 def test_article_page_ignores_related_posts_strip():
     "the other side of #774: the same shape is a related-posts strip, which must not displace the body."
     body = " ".join(

trafilatura/main_extractor.py

@@ -8,7 +8,7 @@ import re  # import regex as re
 from copy import deepcopy
 from urllib.parse import urljoin
 
-from lxml.etree import Element, SubElement, _Element, strip_elements, strip_tags, tostring
+from lxml.etree import Element, SubElement, XPath, _Element, strip_elements, strip_tags, tostring
 from lxml.html import HtmlElement
 
 # own
@@ -645,6 +645,23 @@ def prune_unwanted_sections(
     return tree
 
 
+# A sibling-article container is the body of a listing page, but on an article page the same
+# shape is a teaser strip ("recommended stories") sitting beside the body. Only the listing
+# container carries the bulk of the page, so that share is what tells the two apart.
+LISTING_TEXT_SHARE = 0.6
+
+
+def _holds_most_of_the_text(tree: HtmlElement, expr: XPath) -> bool:
+    "Whether the first element matched by expr carries most of the text of the tree."
+    match = next((s for s in expr(tree) if s is not None), None)
+    if match is None:
+        return False
+    page_length = len(trim(" ".join(tree.itertext())))
+    if not page_length:
+        return False
+    return len(trim(" ".join(match.itertext()))) / page_length >= LISTING_TEXT_SHARE
+
+
 def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[str]]:
     # init
     potential_tags = set(TAG_CATALOG)
@@ -655,8 +672,9 @@ def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[
     if options.links is True:
         potential_tags.add("ref")
     result_body = Element("body")
-    # kept out of the confident passes: see LISTING_BODY_XPATH
+    expressions = BODY_XPATH
-    expressions = BODY_XPATH if options.focus != "recall" else LISTING_BODY_XPATH + BODY_XPATH
+    if options.focus == "recall" and _holds_most_of_the_text(tree, LISTING_BODY_XPATH[0]):
+        expressions = LISTING_BODY_XPATH + BODY_XPATH
     # iterate
     for expr in expressions:
         # select tree if the expression has been found