Use the listing container only when it widens a short first entry

On correctiv.org the article page preloads the next twelve full stories as
sibling <article> elements, so with favor_recall the listing container took
all of them (98k chars instead of 9.6k). A teaser strip beside a short body
(mopo.de) was also taken over the body.

The container is now used only when the element BODY_XPATH picks on its own
is one of its entries and shorter than the recall sure-thing length
(10x min_extracted_size).

tests/unit_tests.py

@@ -2922,6 +2922,37 @@ def test_article_page_ignores_related_posts_strip():
     assert "Related teaser 1" not in result
 
 
+def test_teaser_strip_beside_a_short_body_does_not_replace_it():
+    "a teaser strip can outweigh a short article, but the body sits outside it, so recall keeps the body"
+    body = "<p>The short body of the story, a single paragraph that is still the actual article.</p>"
+    teasers = "".join(
+        f"<article><h2><a href='/r{i}'>Teaser headline {i}</a></h2>"
+        f"<p>Teaser text {i} runs over a clause or two, as a lead paragraph on a strip would.</p></article>"
+        for i in range(12)
+    )
+    doc = (
+        f"<html><body><main><article><h1>The Real Headline</h1>{body}</article>"
+        f"<section>{teasers}</section></main></body></html>"
+    )
+    result = extract(doc, output_format="markdown", favor_recall=True, config=use_config()) or ""
+    assert "The short body of the story" in result
+    assert "Teaser headline 3" not in result
+
+
+@pytest.mark.parametrize("settings", [{}, {"favor_recall": True}, {"favor_recall": True, "fast": True}])
+def test_preloaded_full_articles_are_not_a_listing(settings):
+    "an article page preloading the next full stories (correctiv) must not extract all of them"
+    with open(path.join(TEST_DIR, "cache", "correctiv.org.zusage.html"), "rb") as f:
+        doc = f.read()
+    with _without_listing_expression():
+        expected = extract(doc, config=use_config(), **settings)
+    result = extract(doc, config=use_config(), **settings)
+    assert result == expected
+    assert "Vorweg: Die beteiligten AfD-Politiker" in result
+    assert "Die AfD und der Deutschland-Kurier" not in result
+    assert "Geldstrafe für die AfD" not in result
+
+
 def test_short_document_keeps_structure():
     "regression #896: the baseline rescue must not flatten a valid short extraction it cannot improve on."
     doc = (

trafilatura/main_extractor.py

@@ -662,6 +662,24 @@ def _holds_most_of_the_text(tree: HtmlElement, expr: XPath) -> bool:
     return len(trim(" ".join(match.itertext()))) / page_length >= LISTING_TEXT_SHARE
 
 
+def _is_listing_page(tree: HtmlElement, options: Extractor) -> bool:
+    """Whether the sibling-article container is the body of a listing page. The container only
+    widens what BODY_XPATH would take on its own: that element has to be one of its entries and a
+    short teaser. A body outside the container means a teaser strip beside the article, and an
+    entry that already clears the recall "sure thing" length is a full article, as on pages that
+    preload the next stories below the current one."""
+    listing_expr = LISTING_BODY_XPATH[0]
+    container = next((s for s in listing_expr(tree) if s is not None), None)
+    if container is None:
+        return False
+    entry = next((s for expr in BODY_XPATH for s in expr(tree) if s is not None), None)
+    if entry is None or entry.getparent() is not container:
+        return False
+    if len(trim(" ".join(entry.itertext()))) > options.min_extracted_size * 10:
+        return False
+    return _holds_most_of_the_text(tree, listing_expr)
+
+
 def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[str]]:
     # init
     potential_tags = set(TAG_CATALOG)
@@ -673,7 +691,7 @@ def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[
         potential_tags.add("ref")
     result_body = Element("body")
     expressions = BODY_XPATH
-    if options.focus == "recall" and _holds_most_of_the_text(tree, LISTING_BODY_XPATH[0]):
+    if options.focus == "recall" and _is_listing_page(tree, options):
         expressions = LISTING_BODY_XPATH + BODY_XPATH
     # iterate
     for expr in expressions: