diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 89179fd..de2b917 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -2922,6 +2922,37 @@ def test_article_page_ignores_related_posts_strip(): assert "Related teaser 1" not in result +def test_teaser_strip_beside_a_short_body_does_not_replace_it(): + "a teaser strip can outweigh a short article, but the body sits outside it, so recall keeps the body" + body = "

The short body of the story, a single paragraph that is still the actual article.

" + teasers = "".join( + f"

Teaser headline {i}

" + f"

Teaser text {i} runs over a clause or two, as a lead paragraph on a strip would.

" + for i in range(12) + ) + doc = ( + f"

The Real Headline

{body}
" + f"
{teasers}
" + ) + result = extract(doc, output_format="markdown", favor_recall=True, config=use_config()) or "" + assert "The short body of the story" in result + assert "Teaser headline 3" not in result + + +@pytest.mark.parametrize("settings", [{}, {"favor_recall": True}, {"favor_recall": True, "fast": True}]) +def test_preloaded_full_articles_are_not_a_listing(settings): + "an article page preloading the next full stories (correctiv) must not extract all of them" + with open(path.join(TEST_DIR, "cache", "correctiv.org.zusage.html"), "rb") as f: + doc = f.read() + with _without_listing_expression(): + expected = extract(doc, config=use_config(), **settings) + result = extract(doc, config=use_config(), **settings) + assert result == expected + assert "Vorweg: Die beteiligten AfD-Politiker" in result + assert "Die AfD und der Deutschland-Kurier" not in result + assert "Geldstrafe für die AfD" not in result + + def test_short_document_keeps_structure(): "regression #896: the baseline rescue must not flatten a valid short extraction it cannot improve on." doc = ( diff --git a/trafilatura/main_extractor.py b/trafilatura/main_extractor.py index 9cd8283..38f3a44 100644 --- a/trafilatura/main_extractor.py +++ b/trafilatura/main_extractor.py @@ -662,6 +662,24 @@ def _holds_most_of_the_text(tree: HtmlElement, expr: XPath) -> bool: return len(trim(" ".join(match.itertext()))) / page_length >= LISTING_TEXT_SHARE +def _is_listing_page(tree: HtmlElement, options: Extractor) -> bool: + """Whether the sibling-article container is the body of a listing page. The container only + widens what BODY_XPATH would take on its own: that element has to be one of its entries and a + short teaser. A body outside the container means a teaser strip beside the article, and an + entry that already clears the recall "sure thing" length is a full article, as on pages that + preload the next stories below the current one.""" + listing_expr = LISTING_BODY_XPATH[0] + container = next((s for s in listing_expr(tree) if s is not None), None) + if container is None: + return False + entry = next((s for expr in BODY_XPATH for s in expr(tree) if s is not None), None) + if entry is None or entry.getparent() is not container: + return False + if len(trim(" ".join(entry.itertext()))) > options.min_extracted_size * 10: + return False + return _holds_most_of_the_text(tree, listing_expr) + + def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[str]]: # init potential_tags = set(TAG_CATALOG) @@ -673,7 +691,7 @@ def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[ potential_tags.add("ref") result_body = Element("body") expressions = BODY_XPATH - if options.focus == "recall" and _holds_most_of_the_text(tree, LISTING_BODY_XPATH[0]): + if options.focus == "recall" and _is_listing_page(tree, options): expressions = LISTING_BODY_XPATH + BODY_XPATH # iterate for expr in expressions: