' f'{title}
{text}
diff --git a/tests/unit_tests.py b/tests/unit_tests.py
index b091c4d..411ca1e 100644
--- a/tests/unit_tests.py
+++ b/tests/unit_tests.py
@@ -2779,7 +2779,7 @@ def _listing_page(entries):
articles = "".join(
f' {text}'
f'{title}
{text}
{text}
Body paragraph {i} runs long enough to carry the page on its own, with several " + f"clauses and a couple of sentences so the article clearly outweighs anything beside " + f"it. Nothing here looks like a teaser or like boilerplate to the extractor.
" + for i in range(6) + ) + related = [(f"Recommended story {i}", f"Teaser {i}.") for i in range(3)] + html_doc = _article_with_related_posts(body, related) + + result = extract(html_doc, output_format="markdown", favor_recall=True, config=use_config()) or "" + + assert "Body paragraph 0" in result + assert "Body paragraph 5" in result + assert "Recommended story 0" not in result + + def test_article_page_ignores_related_posts_strip(): "the other side of #774: the same shape is a related-posts strip, which must not displace the body." body = " ".join( diff --git a/trafilatura/main_extractor.py b/trafilatura/main_extractor.py index 95910f8..9cd8283 100644 --- a/trafilatura/main_extractor.py +++ b/trafilatura/main_extractor.py @@ -8,7 +8,7 @@ import re # import regex as re from copy import deepcopy from urllib.parse import urljoin -from lxml.etree import Element, SubElement, _Element, strip_elements, strip_tags, tostring +from lxml.etree import Element, SubElement, XPath, _Element, strip_elements, strip_tags, tostring from lxml.html import HtmlElement # own @@ -645,6 +645,23 @@ def prune_unwanted_sections( return tree +# A sibling-article container is the body of a listing page, but on an article page the same +# shape is a teaser strip ("recommended stories") sitting beside the body. Only the listing +# container carries the bulk of the page, so that share is what tells the two apart. +LISTING_TEXT_SHARE = 0.6 + + +def _holds_most_of_the_text(tree: HtmlElement, expr: XPath) -> bool: + "Whether the first element matched by expr carries most of the text of the tree." + match = next((s for s in expr(tree) if s is not None), None) + if match is None: + return False + page_length = len(trim(" ".join(tree.itertext()))) + if not page_length: + return False + return len(trim(" ".join(match.itertext()))) / page_length >= LISTING_TEXT_SHARE + + def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[str]]: # init potential_tags = set(TAG_CATALOG) @@ -655,8 +672,9 @@ def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[ if options.links is True: potential_tags.add("ref") result_body = Element("body") - # kept out of the confident passes: see LISTING_BODY_XPATH - expressions = BODY_XPATH if options.focus != "recall" else LISTING_BODY_XPATH + BODY_XPATH + expressions = BODY_XPATH + if options.focus == "recall" and _holds_most_of_the_text(tree, LISTING_BODY_XPATH[0]): + expressions = LISTING_BODY_XPATH + BODY_XPATH # iterate for expr in expressions: # select tree if the expression has been found