dualfroz
ce36d8e544
A blog index, archive or category page holds several <article> siblings. BODY_XPATH's "(.//article)[1]" keeps the first entry only, so the page looks under-extracted and escalates to justext, which emits paragraphs and drops every heading. Give the recall pass a listing expression that takes the container of the sibling articles, so the retry returns all entries with their heads and wins the escalation. Recall only: on an article page the same shape is a related-posts strip, whose container would hide the body.
490 lines
17 KiB
Python
490 lines
17 KiB
Python
# pylint:disable-msg=E0611
|
|
"""
|
|
X-Path expressions used to extract or filter the main text content,
|
|
and to extract metadata.
|
|
"""
|
|
|
|
from lxml.etree import XPath
|
|
|
|
REGEXP_NS = "http://exslt.org/regular-expressions"
|
|
|
|
|
|
def _alt(tokens: tuple[str, ...]) -> str:
|
|
"Join concept tokens into a regex alternation; reject an empty group (would match every element)."
|
|
if not tokens: # not an assert: must hold under python -O
|
|
raise ValueError("empty token group would make re:test(...) match everything")
|
|
return "|".join(tokens)
|
|
|
|
|
|
### 1. CONTENT
|
|
|
|
# Content-area token vocabulary for BODY_XPATH, grouped per stage, composed via _alt() into the
|
|
# re:test(...) alternations below.
|
|
_ARTICLE_CONTENT_ID_TOKENS = (
|
|
"(?:entry|article|art)-content",
|
|
"article__content",
|
|
"article(?:-|__)?body",
|
|
"articleBody",
|
|
"body-text",
|
|
)
|
|
_ARTICLE_CONTENT_CLASS_TOKENS = (
|
|
"post[-_]text",
|
|
"post-body",
|
|
"post-?entry",
|
|
"post[-_]?content",
|
|
"postContent",
|
|
"post_inner_wrapper",
|
|
"article-?text",
|
|
"articleText",
|
|
"(?:entry|page|text|article|art)-content",
|
|
"article__content",
|
|
"article(?:-|__)?body",
|
|
"articleBody",
|
|
"ArticleContent",
|
|
"body-text",
|
|
"article__container",
|
|
)
|
|
_STORY_ID_TOKENS = ("^primary", "story-body")
|
|
_STORY_CLASS_TOKENS = (
|
|
"^article ",
|
|
"post-bodycopy",
|
|
"story-?content",
|
|
"(?:theme|blog|section|single)-content",
|
|
"single-post",
|
|
"main-column",
|
|
"wpb_text_column",
|
|
"story-body",
|
|
"field-body",
|
|
)
|
|
_MAIN_CONTENT_ID_TOKENS = ("content-main", "content-body", "contentBody")
|
|
_MAIN_CONTENT_CLASS_TOKENS = ("content[-_]main", "content(?:-|__)body")
|
|
|
|
BODY_XPATH = [
|
|
XPath(
|
|
f"""
|
|
.//*[self::article or self::div or self::main or self::section][
|
|
@class='post' or @class='entry' or
|
|
@itemprop='articleBody' or @id='articleContent' or
|
|
re:test(@id, '{_alt(_ARTICLE_CONTENT_ID_TOKENS)}') or
|
|
re:test(@class, '{_alt(_ARTICLE_CONTENT_CLASS_TOKENS)}')
|
|
][1]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
# (…)[1] = first occurrence
|
|
XPath("(.//article)[1]"),
|
|
XPath(
|
|
f"""
|
|
(.//*[self::article or self::div or self::main or self::section][
|
|
@role='article' or
|
|
@id='article' or @id='story' or
|
|
@class='postarea' or @class='art-postcontent' or @class='text' or @class='cell' or @class='story' or
|
|
re:test(@id, '{_alt(_STORY_ID_TOKENS)}') or
|
|
re:test(@class, 'fulltext', 'i') or
|
|
re:test(@class, '{_alt(_STORY_CLASS_TOKENS)}')
|
|
])[1]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath(
|
|
f"""
|
|
(.//*[self::article or self::div or self::main or self::section][
|
|
@id='content' or @class='content' or
|
|
re:test(@id, '{_alt(_MAIN_CONTENT_ID_TOKENS)}') or
|
|
re:test(@class, '{_alt(_MAIN_CONTENT_CLASS_TOKENS)}') or
|
|
contains(translate(@id, 'CM','cm'), 'main-content') or contains(translate(@class, 'CM','cm'), 'main-content') or
|
|
contains(translate(@class, 'CP','cp'), 'page-content')
|
|
])[1]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath(
|
|
"""
|
|
(.//*[self::article or self::div or self::section][
|
|
starts-with(@class, 'main') or starts-with(@id, 'main') or starts-with(@role, 'main')])[1]|(.//main)[1]
|
|
"""
|
|
),
|
|
]
|
|
|
|
# Listing layouts (blog index, archive, category) hold several <article> siblings in one container,
|
|
# each a teaser with its own heading. BODY_XPATH's "(.//article)[1]" keeps the first entry only, so
|
|
# the page escalates to justext, which flattens every heading into a paragraph (#774). Recall only:
|
|
# on an article page the same shape is a related-posts strip, whose container would hide the body.
|
|
LISTING_BODY_XPATH = [XPath("(.//article[preceding-sibling::article])[1]/..")]
|
|
|
|
# starts-with(@id, "article") or
|
|
# or starts-with(@id, "story") or contains(@class, "story")
|
|
# starts-with(@class, "content ") or contains(@class, " content")
|
|
# '//div[contains(@class, "text") or contains(@class, "article-wrapper") or contains(@class, "content-wrapper")]',
|
|
# '//div[contains(@class, "article-wrapper") or contains(@class, "content-wrapper")]',
|
|
# |//*[self::article or self::div or self::main or self::section][contains(@class, "article") or contains(@class, "Article")]
|
|
# @id="content"or @class="content" or @class="Content"
|
|
# or starts-with(@class, 'post ')
|
|
# './/span[@class=""]', # instagram?
|
|
|
|
|
|
COMMENTS_XPATH = [
|
|
XPath(
|
|
"""
|
|
.//*[self::div or self::list or self::section][
|
|
re:test(@id|@class, 'comment-?list') or
|
|
re:test(@class, 'comment-page|comments-content|post-comments')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath(
|
|
"""
|
|
.//*[self::div or self::section or self::list][
|
|
re:test(@id|@class, '^comment[s-]') or
|
|
re:test(@class, '^Comments|article-comments')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath(
|
|
"""
|
|
.//*[self::div or self::section or self::list][
|
|
re:test(@id, '^(?:comol|disqus_thread|dsq-comments)')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath(
|
|
"""
|
|
.//*[self::div or self::section][
|
|
starts-with(@id, 'social') or contains(@class, 'comment')]
|
|
"""
|
|
),
|
|
]
|
|
# or contains(@class, 'Comments')
|
|
|
|
_COMMENTS_ID = "[Cc]omment(?!ary)|comol"
|
|
_COMMENTS_CLASS = "^[Cc]omment(?!ary)|(?:article|post)-comments"
|
|
REMOVE_COMMENTS_XPATH = [
|
|
XPath(
|
|
f"""
|
|
.//*[self::div or self::list or self::section or self::details][
|
|
re:test(@id, '^(?:{_COMMENTS_ID}|disqus_thread|dsq-comments)') or
|
|
re:test(@class, '{_COMMENTS_CLASS}')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
)
|
|
]
|
|
# plus ol/ul comment lists, without the disqus ids, in one tree walk
|
|
REMOVE_COMMENTS_AND_LISTS_XPATH = [
|
|
XPath(
|
|
f"""
|
|
.//*[self::div or self::list or self::section or self::details or self::ol or self::ul][
|
|
re:test(@id, '^(?:{_COMMENTS_ID}|disqus_thread|dsq-comments)') or
|
|
re:test(@class, '{_COMMENTS_CLASS}')][
|
|
not(self::ol or self::ul) or re:test(@id, '^(?:{_COMMENTS_ID})') or re:test(@class, '{_COMMENTS_CLASS}')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
)
|
|
]
|
|
# or self::span
|
|
# or contains(@class, 'comment') or contains(@id, 'comment')
|
|
|
|
# Raw-tree prune, kept out of OVERALL_DISCARD_XPATH because its over-pruning guard would revert it.
|
|
RAW_TREE_PRUNE_XPATH = [
|
|
# infinite-scroll containers holding whole follow-up articles ("mvp-post-add": Zox News theme),
|
|
# such pages are mostly appended articles, a generic container holding the h1 is the real one
|
|
XPath(
|
|
"""
|
|
.//*[self::div or self::section or self::aside][
|
|
re:test(@id, 'mvp-post-add-(?:box|wrap)|infinite-?scroll') or
|
|
re:test(@class, 'mvp-post-add-(?:box|wrap)|infinite-?scroll')][
|
|
not(.//h1) or re:test(@id, 'mvp-post-add-(?:box|wrap)') or re:test(@class, 'mvp-post-add-(?:box|wrap)')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
# OVERALL_DISCARD_XPATH's "share-" token matches these already, but its "widget" token
|
|
# also matches the article body ("elementor-widget")
|
|
XPath(".//*[self::div or self::section or self::aside][contains(@class, 'elementor-share-buttons')]"),
|
|
]
|
|
|
|
|
|
# OVERALL_DISCARD_XPATH boilerplate token vocabulary, grouped by concept, composed via _alt()
|
|
# below. _LEGACY_SITE_* = single-site/legacy tokens (provenance + per-token audit in memory).
|
|
_SHARE_SOCIAL_ID_CLASS_TOKENS = ("^shar", "social", "viral")
|
|
_NEWSLETTER_ID_CLASS_TOKENS = ("newsletter", "syndication")
|
|
_CONSENT_ID_CLASS_TOKENS = ("cookie",)
|
|
_TAGS_ID_CLASS_TOKENS = ("tags",)
|
|
_UI_CHROME_ID_CLASS_TOKENS = ("sidebar", "banner", "bread-?crumb", "button")
|
|
_AUTHOR_ID_CLASS_TOKENS = ("author",)
|
|
|
|
_LEGACY_SITE_ID_TOKENS = ("^(?:jp-|dpsp-content)", "bmdh")
|
|
_FOOTER_ID_TOKENS = ("footer", "Footer")
|
|
_SHARE_ID_TOKENS = ("share", "Share")
|
|
_NAV_ID_TOKENS = ("nav", "Nav", "menu")
|
|
_RELATED_ID_TOKENS = ("related",)
|
|
_UI_CHROME_ID_TOKENS = ("message-container",)
|
|
_PAYWALL_ID_TOKENS = ("premium",)
|
|
_AD_ID_TOKENS = ("^ad-",)
|
|
|
|
_NAV_CLASS_TOKENS = ("^(?:nav|post-nav|ZendeskForm)", "subnav", "avigation", "navbar", "navbox", "menu", "bar")
|
|
_AD_CLASS_TOKENS = (
|
|
" ad ",
|
|
"-ad-",
|
|
"(?:^| )ad-",
|
|
"dfp[Aa]d",
|
|
"outbrain",
|
|
"taboola",
|
|
"criteo",
|
|
"paid-?content",
|
|
"widget",
|
|
)
|
|
_FOOTER_CLASS_TOKENS = ("footer", "Footer")
|
|
_AUTHOR_CLASS_TOKENS = ("byline", "Byline")
|
|
_SHARE_CLASS_TOKENS = ("share-", "sociable", "embedded", "embed")
|
|
_TAGS_CLASS_TOKENS = ("tag-list",)
|
|
_CONSENT_CLASS_TOKENS = ("consent", "modal-content", "permission")
|
|
_RELATED_CLASS_TOKENS = ("elated", "next-", "-stories", "most-popular", "more-on")
|
|
_UI_META_CLASS_TOKENS = (
|
|
"meta",
|
|
"rating",
|
|
"attachment",
|
|
"timestamp",
|
|
"user-info",
|
|
"user-profile",
|
|
"-icon",
|
|
"article-infos",
|
|
"message-container",
|
|
"slide",
|
|
"viewport",
|
|
"overlay",
|
|
)
|
|
_MISC_CLASS_TOKENS = ("options", "expand", "obfuscated", "blurred")
|
|
_LEGACY_SITE_CLASS_TOKENS = ("mol-factbox", "yin", "zlylin", "nfoline")
|
|
|
|
# id/class concept tokens, checked on BOTH attributes in the xpath below. NOTE re:test(@id|@class,...)
|
|
# tests only the SOURCE-FIRST attribute (XPath string(node-set)=first node), so each is applied
|
|
# per-attribute. 'cookie' (_CONSENT_ID_CLASS_TOKENS) is deliberately kept first-attr-only (via the
|
|
# @id|@class clause): pages ABOUT cookies carry the token on real content, so both-attr over-discards.
|
|
_OVERALL_DISCARD_BOTH_TOKENS = (
|
|
_SHARE_SOCIAL_ID_CLASS_TOKENS
|
|
+ _NEWSLETTER_ID_CLASS_TOKENS
|
|
+ _TAGS_ID_CLASS_TOKENS
|
|
+ _UI_CHROME_ID_CLASS_TOKENS
|
|
+ _AUTHOR_ID_CLASS_TOKENS
|
|
)
|
|
_OVERALL_DISCARD_ID_TOKENS = (
|
|
_LEGACY_SITE_ID_TOKENS
|
|
+ _FOOTER_ID_TOKENS
|
|
+ _SHARE_ID_TOKENS
|
|
+ _NAV_ID_TOKENS
|
|
+ _RELATED_ID_TOKENS
|
|
+ _UI_CHROME_ID_TOKENS
|
|
+ _PAYWALL_ID_TOKENS
|
|
+ _AD_ID_TOKENS
|
|
)
|
|
_OVERALL_DISCARD_CLASS_TOKENS = (
|
|
_NAV_CLASS_TOKENS
|
|
+ _AD_CLASS_TOKENS
|
|
+ _FOOTER_CLASS_TOKENS
|
|
+ _AUTHOR_CLASS_TOKENS
|
|
+ _SHARE_CLASS_TOKENS
|
|
+ _TAGS_CLASS_TOKENS
|
|
+ _CONSENT_CLASS_TOKENS
|
|
+ _RELATED_CLASS_TOKENS
|
|
+ _UI_META_CLASS_TOKENS
|
|
+ _MISC_CLASS_TOKENS
|
|
+ _LEGACY_SITE_CLASS_TOKENS
|
|
)
|
|
|
|
OVERALL_DISCARD_XPATH = [
|
|
XPath(
|
|
f"""
|
|
.//*[self::div or self::item or self::list or self::p or self::section or self::span][
|
|
@data-lp-replacement-content or
|
|
contains(translate(@role, 'N', 'n'), 'nav') or
|
|
contains(@data-component, 'MostPopularStories') or
|
|
re:test(@id|@class, '{_alt(_CONSENT_ID_CLASS_TOKENS)}') or
|
|
re:test(@id, '{_alt(_OVERALL_DISCARD_BOTH_TOKENS + _OVERALL_DISCARD_ID_TOKENS)}') or
|
|
re:test(@class, '{_alt(_OVERALL_DISCARD_BOTH_TOKENS + _OVERALL_DISCARD_CLASS_TOKENS)}')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath(
|
|
"""
|
|
.//*[@class='comments-title' or
|
|
starts-with(@id|@class, 'reply-') or
|
|
re:test(@id|@style, 'hidden') or
|
|
contains(@style, 'display:none') or contains(@style, 'display: none') or
|
|
re:test(@id, 'reader-comments|akismet') or
|
|
re:test(@class, '^hide-|comments-title|nocomments|-reply-|message|akismet|suggest-links|-hide-|hide-print| hidden| hide|noprint|notloaded') or @aria-hidden='true']
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
]
|
|
|
|
# conflicts:
|
|
# contains(@id, "header") or contains(@class, "header") or
|
|
# class contains "cats" (categories, also tags?)
|
|
# or contains(@class, "hidden ") or contains(@class, "-hide")
|
|
# or contains(@class, "paywall")
|
|
# contains(@class, "content-info") or contains(@class, "content-title")
|
|
# contains(translate(@class, "N", "n"), "nav") or
|
|
# contains(@class, "panel") or
|
|
# or starts-with(@id, "comment-")
|
|
|
|
|
|
# the following conditions focus on extraction precision
|
|
TEASER_DISCARD_XPATH = [
|
|
XPath(
|
|
"""
|
|
.//*[self::div or self::item or self::list or self::p or self::section or self::span][
|
|
contains(translate(@id, 'T', 't'), 'teaser') or contains(translate(@class, 'T', 't'), 'teaser')]
|
|
"""
|
|
)
|
|
]
|
|
|
|
|
|
PRECISION_DISCARD_XPATH = [
|
|
XPath(""".//header"""),
|
|
# 'link' matched as a whole class token, not a substring (that dropped permalink/headline-link/
|
|
# etc.); still drops class="link" (guarded by test_precision_recall). 'border' = substring.
|
|
# 'bottom' must start or end a token: page-bottom chrome is named 'bottom'/'bottom-bar'/
|
|
# 'article-bottom', whereas a mid-token 'bottom' is a CSS spacing utility
|
|
# ('Padding-bottom-lg-30', 'border-bottom-0') that layout wrappers put on real content.
|
|
XPath(
|
|
r"""
|
|
.//*[self::div or self::item or self::list or self::p or self::section or self::span][
|
|
re:test(@id|@class, '(^|\s)bottom|bottom(\s|$)') or re:test(@id|@class, '(^|\s)link(\s|$)') or
|
|
contains(@style, 'border')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
]
|
|
# or contains(@id, "-comments") or contains(@class, "-comments")
|
|
|
|
|
|
DISCARD_IMAGE_ELEMENTS = [
|
|
XPath(
|
|
"""
|
|
.//*[self::div or self::item or self::list or self::p or self::section or self::span][
|
|
contains(@id, 'caption') or contains(@class, 'caption')]
|
|
"""
|
|
)
|
|
]
|
|
|
|
|
|
COMMENTS_DISCARD_XPATH = [
|
|
XPath(""".//*[self::div or self::section][starts-with(@id, 'respond')]"""),
|
|
XPath(""".//cite|.//quote"""),
|
|
XPath(
|
|
"""
|
|
.//*[
|
|
@class='comments-title' or
|
|
contains(@style, 'display:none') or
|
|
re:test(@class, 'comments-title|nocomments|-reply-|message|signin') or
|
|
re:test(@id|@class, '^reply-|akismet')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
]
|
|
|
|
|
|
### 2. METADATA
|
|
|
|
|
|
# the order or depth of XPaths could be changed after exhaustive testing
|
|
AUTHOR_XPATHS = [
|
|
XPath(
|
|
# specific and almost specific
|
|
"""
|
|
//*[self::a or self::address or self::div or self::link or self::p or self::span or self::strong][
|
|
@rel='author' or @id='author' or @class='author' or @itemprop='author name' or rel='me' or
|
|
@data-testid='AuthorCard' or @data-testid='AuthorURL' or
|
|
re:test(@class, 'author-?name|AuthorName|authorName')]|//author
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath(
|
|
# almost generic and generic, last ones not common
|
|
"""
|
|
//*[self::a or self::div or self::h3 or self::h4 or self::p or self::span][
|
|
@class='byline' or @class='username' or @class='byl' or @class='BBL' or
|
|
contains(@itemprop, 'author') or
|
|
re:test(@id, 'author|zuozhe|bianji|xiaobian') or
|
|
re:test(@class, 'author|channel-name|zuozhe|bianji|xiaobian|submitted-by|posted-by|journalist-name')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath(
|
|
# last resort: any element
|
|
"""
|
|
//*[
|
|
contains(@data-component, 'Byline') or contains(@itemprop, 'author') or
|
|
re:test(@id, '[Aa]uthor') or
|
|
re:test(@class, '[Aa]uthor|screenname|writer|[Bb]yline')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
]
|
|
|
|
|
|
AUTHOR_DISCARD_XPATHS = [
|
|
XPath(
|
|
"""
|
|
.//*[self::a or self::div or self::section or self::span][
|
|
@id='comments' or @class='comments' or @class='title' or @class='date' or
|
|
re:test(@id, '^comments|comment-?list|ProductReviews') or
|
|
re:test(@class, '^[Cc]omments|commentlist|comments-list|sidebar|is-hidden|quote|embedly-instagram|article-(?:share|support)|print|category|meta-date|meta-reviewer') or
|
|
contains(@data-component, 'Figure')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath("//time|//figure"),
|
|
]
|
|
|
|
|
|
CATEGORIES_XPATHS = [
|
|
XPath(
|
|
"""
|
|
//div[
|
|
re:test(@class, '^(?:post-?info|post-?meta|meta|entry-meta|entry-info|entry-utility)') or
|
|
starts-with(@id, 'postpath')]//a[@href]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath("""//p[starts-with(@class, 'postmeta') or starts-with(@class, 'entry-categories') or
|
|
@class='postinfo' or @id='filedunder']//a[@href]"""),
|
|
XPath("""//footer[starts-with(@class, 'entry-meta') or starts-with(@class, 'entry-footer')]//a[@href]"""),
|
|
XPath("""//*[self::li or self::span][@class='post-category' or @class='postcategory' or
|
|
@class='entry-category' or contains(@class, 'cat-links')]//a[@href]"""),
|
|
XPath("""//header[@class='entry-header']//a[@href]"""),
|
|
XPath("""//div[@class='row' or @class='tags']//a[@href]"""),
|
|
]
|
|
# "//*[self::div or self::p][contains(@class, 'byline')]",
|
|
|
|
|
|
TAGS_XPATHS = [
|
|
XPath("""//div[@class='tags']//a[@href]"""),
|
|
XPath("""//p[starts-with(@class, 'entry-tags')]//a[@href]"""),
|
|
XPath(
|
|
"""//div[@class='row' or @class='jp-relatedposts' or @class='entry-utility' or
|
|
re:test(@class, '^(?:tag|postmeta|meta)')]//a[@href]""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath("""//*[@class='entry-meta' or contains(@class, 'topics') or
|
|
contains(@class, 'tags-links')]//a[@href]"""),
|
|
]
|
|
# "related-topics"
|
|
# https://github.com/grangier/python-goose/blob/develop/goose/extractors/tags.py
|
|
|
|
|
|
TITLE_XPATHS = [
|
|
XPath(
|
|
"""
|
|
//*[self::h1 or self::h2][
|
|
re:test(@class, '(?:post-|entry-|article-|post__)title|headline') or
|
|
contains(@id, 'headline') or contains(@itemprop, 'headline')]
|
|
""",
|
|
namespaces={"re": REGEXP_NS},
|
|
),
|
|
XPath("""//*[@class='entry-title' or @class='post-title']"""),
|
|
XPath("""//*[self::h1 or self::h2 or self::h3][
|
|
contains(@class, 'title') or contains(@id, 'title')]"""),
|
|
]
|
|
# json-ld headline
|
|
# '//header/h1',
|