fix: keep every entry heading on listing pages (#774)
A blog index, archive or category page holds several <article> siblings. BODY_XPATH's "(.//article)[1]" keeps the first entry only, so the page looks under-extracted and escalates to justext, which emits paragraphs and drops every heading. Give the recall pass a listing expression that takes the container of the sibling articles, so the retry returns all entries with their heads and wins the escalation. Recall only: on an article page the same shape is a related-posts strip, whose container would hide the body.
This commit is contained in:
1 parent
7568e2a786
commit
ce36d8e544
3 files changed
+56
-1
No files matched your search
@@ -35,6 +35,7 @@ from .xpaths import (
|
||||
COMMENTS_DISCARD_XPATH,
|
||||
COMMENTS_XPATH,
|
||||
DISCARD_IMAGE_ELEMENTS,
|
||||
LISTING_BODY_XPATH,
|
||||
OVERALL_DISCARD_XPATH,
|
||||
PRECISION_DISCARD_XPATH,
|
||||
TEASER_DISCARD_XPATH,
|
||||
@@ -654,8 +655,10 @@ def _extract(tree: HtmlElement, options: Extractor) -> tuple[_Element, str, set[
|
||||
if options.links is True:
|
||||
potential_tags.add("ref")
|
||||
result_body = Element("body")
|
||||
# kept out of the confident passes: see LISTING_BODY_XPATH
|
||||
expressions = BODY_XPATH if options.focus != "recall" else LISTING_BODY_XPATH + BODY_XPATH
|
||||
# iterate
|
||||
for expr in BODY_XPATH:
|
||||
for expr in expressions:
|
||||
# select tree if the expression has been found
|
||||
subtree = next((s for s in expr(tree) if s is not None), None)
|
||||
if subtree is None:
|
||||
|
||||
@@ -105,6 +105,13 @@ BODY_XPATH = [
|
||||
"""
|
||||
),
|
||||
]
|
||||
|
||||
# Listing layouts (blog index, archive, category) hold several <article> siblings in one container,
|
||||
# each a teaser with its own heading. BODY_XPATH's "(.//article)[1]" keeps the first entry only, so
|
||||
# the page escalates to justext, which flattens every heading into a paragraph (#774). Recall only:
|
||||
# on an article page the same shape is a related-posts strip, whose container would hide the body.
|
||||
LISTING_BODY_XPATH = [XPath("(.//article[preceding-sibling::article])[1]/..")]
|
||||
|
||||
# starts-with(@id, "article") or
|
||||
# or starts-with(@id, "story") or contains(@class, "story")
|
||||
# starts-with(@class, "content ") or contains(@class, " content")
|
||||
|
||||
Reference in new issue
Block a user