diff --git a/tests/unit_tests.py b/tests/unit_tests.py index 977d4f5..b091c4d 100644 --- a/tests/unit_tests.py +++ b/tests/unit_tests.py @@ -2772,6 +2772,51 @@ def test_no_duplicate_content(): assert out.count("Second synthetic paragraph") == 1 +def _listing_page(entries): + "Blog-index markup: sibling
teasers, each a link-only heading plus its lead paragraph." + # each teaser clears min_extracted_size on its own, so only the page-share + # test flags the layout as under-extracted and escalates to recall + articles = "".join( + f'

' + f'{title}

' + f"

{text}

" + for i, (title, text) in enumerate(entries) + ) + return f'
{articles}
' + + +def test_listing_page_keeps_every_entry_heading(): + "regression #774: an index page must not collapse to its first entry, losing every other heading." + entries = [ + ( + f"Entry number {i} about a self contained subject", + f"Teaser {i} carries enough prose to pass the extractor thresholds on its own, running " + f"over several clauses and a couple of sentences. It reads as a lead paragraph would, " + f"so nothing about this entry looks like boilerplate to the extractor, and the index " + f"page as a whole reads as a real body of text rather than a stub.", + ) + for i in range(8) + ] + result = extract(_listing_page(entries), output_format="markdown", config=use_config()) or "" + for title, _ in entries: + assert f"## {title}" in result + + +def test_article_page_ignores_related_posts_strip(): + "the other side of #774: the same shape is a related-posts strip, which must not displace the body." + body = " ".join( + f"Sentence {i} of the actual article body, long enough that the extractor never escalates." for i in range(8) + ) + related = "".join(f"

Related teaser {i}

" for i in range(3)) + doc = ( + f'

The Real Headline

{body}

' + f'
' + ) + result = extract(doc, output_format="txt", config=use_config()) or "" + assert "Sentence 3 of the actual article body" in result + assert "Related teaser 1" not in result + + def test_short_document_keeps_structure(): "regression #896: the baseline rescue must not flatten a valid short extraction it cannot improve on." doc = ( diff --git a/trafilatura/main_extractor.py b/trafilatura/main_extractor.py index c8c0ac1..95910f8 100644 --- a/trafilatura/main_extractor.py +++ b/trafilatura/main_extractor.py @@ -35,6 +35,7 @@ from .xpaths import ( COMMENTS_DISCARD_XPATH, COMMENTS_XPATH, DISCARD_IMAGE_ELEMENTS, + LISTING_BODY_XPATH, OVERALL_DISCARD_XPATH, PRECISION_DISCARD_XPATH, TEASER_DISCARD_XPATH, @@ -654,8 +655,10 @@ def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[ if options.links is True: potential_tags.add("ref") result_body = Element("body") + # kept out of the confident passes: see LISTING_BODY_XPATH + expressions = BODY_XPATH if options.focus != "recall" else LISTING_BODY_XPATH + BODY_XPATH # iterate - for expr in BODY_XPATH: + for expr in expressions: # select tree if the expression has been found subtree = next((s for s in expr(tree) if s is not None), None) if subtree is None: diff --git a/trafilatura/xpaths.py b/trafilatura/xpaths.py index 11ca7ba..a343256 100644 --- a/trafilatura/xpaths.py +++ b/trafilatura/xpaths.py @@ -105,6 +105,13 @@ BODY_XPATH = [ """ ), ] + +# Listing layouts (blog index, archive, category) hold several
siblings in one container, +# each a teaser with its own heading. BODY_XPATH's "(.//article)[1]" keeps the first entry only, so +# the page escalates to justext, which flattens every heading into a paragraph (#774). Recall only: +# on an article page the same shape is a related-posts strip, whose container would hide the body. +LISTING_BODY_XPATH = [XPath("(.//article[preceding-sibling::article])[1]/..")] + # starts-with(@id, "article") or # or starts-with(@id, "story") or contains(@class, "story") # starts-with(@class, "content ") or contains(@class, " content")