Keep the listing container off forum threads

Thread pages line up their posts as sibling <article> elements too, so in
recall mode the container took every post along with its user card and
quotes (katzen-forum.net, XenForo).

The container is now used only when every entry is a teaser: its first
heading links to a page of its own, which no other entry links to. Poster
names come back along a thread or carry no link. On top of that it is never
used on pages already detected as thread forums (DiscussionForumPosting).
This commit is contained in:
dualfroz committed 2026-10-10 15:26:37 +02:00
1 parent 03e823a030
commit 77465a50e7
3 files changed
+91 -12

No files matched your search

+56
View File
@@ -2953,6 +2953,62 @@ def test_preloaded_full_articles_are_not_a_listing(settings):
assert "Geldstrafe für die AfD" not in result
def test_listing_page_with_links_keeps_every_entry_heading():
"with links kept the heading targets live in another attribute, the listing must still be found"
entries = _listing_entries()
doc = _listing_page(entries)
result = extract(doc, output_format="markdown", include_links=True, config=use_config()) or ""
for i, (title, text) in enumerate(entries):
assert f"## [{title}](/{i})" in result
assert text in result
def _forum_thread(posters, linked=True, forum_markup=False):
"Thread markup: sibling <article> posts, each opening with a user card that names its poster in a heading."
name_markup = '<a href="/members/{0}/">{0}</a>' if linked else "<span>{0}</span>"
posts = "".join(
f'<article><div class="poster"><h4>{name_markup.format(name)}</h4>'
"<dl><dt>Joined</dt><dd>March 2015</dd><dt>Messages</dt><dd>1,447</dd></dl></div>"
f"<div><p>Reply {i} from {name} carries enough prose to count as a post, running over a couple "
"of clauses so the thread reads as a real discussion between people.</p></div></article>"
for i, name in enumerate(posters)
)
json_ld = (
'<script type="application/ld+json">{"@context": "https://schema.org", '
'"@type": "DiscussionForumPosting", "headline": "A thread"}</script>'
if forum_markup
else ""
)
return (
f"<html><head>{json_ld}</head><body><div id='wrapper'><main>"
f"<div class='thread'>{posts}</div></main></div></body></html>"
)
@pytest.mark.parametrize(
"posters,linked,forum_markup",
[
# the same poster comes back along the thread
(["alice", "bob", "alice", "carol", "bob", "alice"], True, False),
# poster names without a link
(["alice", "bob", "carol", "dave", "erin", "frank"], False, False),
# every poster different, but the page declares itself a forum thread
(["alice", "bob", "carol", "dave", "erin", "frank"], True, True),
],
)
@pytest.mark.parametrize("settings", [{"favor_recall": True}, {"favor_recall": True, "fast": True}])
def test_forum_thread_is_not_a_listing(posters, linked, forum_markup, settings):
"forum posts line up like listing entries, but taking their container brings every user card along"
doc = _forum_thread(posters, linked, forum_markup)
with _without_listing_expression():
expected = extract(doc, output_format="markdown", config=use_config(), **settings)
result = extract(doc, output_format="markdown", config=use_config(), **settings) or ""
assert result == expected
assert result.count("Joined") < len(posters)
for i in range(len(posters)):
assert f"Reply {i} from" in result
def test_short_document_keeps_structure():
"regression #896: the baseline rescue must not flatten a valid short extraction it cannot improve on."
doc = (