commit 6675a8b88470234156d4aae0891f395d9ab6004e Author: upstream Date: Sat Oct 10 12:37:20 2026 +0200 Upstream scrapy/scrapy state of the touched files before c9f94dc564 diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py new file mode 100644 index 0000000..210d525 --- /dev/null +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -0,0 +1,142 @@ +""" +This is a middleware to respect robots.txt policies. To activate it you must +enable this middleware and enable the ROBOTSTXT_OBEY setting. + +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any, Self + +from twisted.internet.defer import Deferred + +from scrapy import signals +from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.http import Request, Response +from scrapy.http.request import NO_CALLBACK +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import build_from_crawler, load_object + +if TYPE_CHECKING: + from scrapy import Spider + from scrapy.crawler import Crawler + from scrapy.robotstxt import RobotParser + from scrapy.statscollectors import StatsCollector + + +logger = logging.getLogger(__name__) + + +class RobotsTxtMiddleware: + DOWNLOAD_PRIORITY: int = 1000 + + def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("ROBOTSTXT_OBEY"): + raise NotConfigured + self._default_useragent: str = crawler.settings["USER_AGENT"] + self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] + self.crawler: Crawler = crawler + self._stats: StatsCollector = crawler.stats + self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {} + self._parserimpl: type[RobotParser] = load_object( + crawler.settings.get("ROBOTSTXT_PARSER") + ) + + # check if parser dependencies are met, this should throw an error otherwise. + build_from_crawler(self._parserimpl, self.crawler, b"") + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + @_warn_spider_arg + async def process_request( + self, request: Request, spider: Spider | None = None + ) -> None: + if request.meta.get("dont_obey_robotstxt"): + return + if request.url.startswith("data:") or request.url.startswith("file:"): + return + rp = await self.robot_parser(request) + self.process_request_2(rp, request) + + def process_request_2(self, rp: RobotParser | None, request: Request) -> None: + if rp is None: + return + + useragent: str | bytes | None = self._robotstxt_useragent + if not useragent: + useragent = request.headers.get(b"User-Agent", self._default_useragent) + assert useragent is not None + if not rp.allowed(request.url, useragent): + logger.debug( + "Forbidden by robots.txt: %(request)s", + {"request": request}, + extra={"spider": self.crawler.spider}, + ) + self._stats.inc_value("robotstxt/forbidden") + raise IgnoreRequest("Forbidden by robots.txt") + + async def robot_parser(self, request: Request) -> RobotParser | None: + url = urlparse_cached(request) + netloc = url.netloc + + if netloc not in self._parsers: + self._parsers[netloc] = Deferred() + robotsurl = f"{url.scheme}://{url.netloc}/robots.txt" + meta: dict[str, Any] = { + "dont_obey_robotstxt": True, + "is_robotstxt_request": True, + } + if "download_handler" in request.meta: + meta["download_handler"] = request.meta["download_handler"] + robotsreq = Request( + robotsurl, + priority=self.DOWNLOAD_PRIORITY, + meta=meta, + callback=NO_CALLBACK, + ) + try: + resp = await self.crawler.engine.download_async(robotsreq) + await self._parse_robots(resp, netloc, request) + except Exception as e: + if not isinstance(e, IgnoreRequest): + logger.exception( + f"Error downloading {request}", + extra={"spider": self.crawler.spider}, + ) + self._robots_error(e, netloc) + self._stats.inc_value("robotstxt/request_count") + + parser = self._parsers[netloc] + if isinstance(parser, Deferred): + return await maybe_deferred_to_future(parser) + return parser + + async def _parse_robots( + self, response: Response, netloc: str, request: Request + ) -> None: + self._stats.inc_value("robotstxt/response_count") + self._stats.inc_value(f"robotstxt/response_status_count/{response.status}") + rp = build_from_crawler(self._parserimpl, self.crawler, response.body) + await self.crawler.signals.send_catch_log_async( + signal=signals.robots_parsed, + robotparser=rp, + request=request, + ) + rp_dfd = self._parsers[netloc] + assert isinstance(rp_dfd, Deferred) + self._parsers[netloc] = rp + rp_dfd.callback(rp) + + def _robots_error(self, exc: Exception, netloc: str) -> None: + if not isinstance(exc, IgnoreRequest): + key = f"robotstxt/exception_count/{type(exc)}" + self._stats.inc_value(key) + rp_dfd = self._parsers[netloc] + assert isinstance(rp_dfd, Deferred) + self._parsers[netloc] = None + rp_dfd.callback(None) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py new file mode 100644 index 0000000..62b91e9 --- /dev/null +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +import asyncio +from unittest import mock + +import pytest +from twisted.internet.defer import Deferred, DeferredList +from twisted.python import failure + +from scrapy import signals +from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware +from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured +from scrapy.http import Request, Response, TextResponse +from scrapy.http.request import NO_CALLBACK +from scrapy.settings import Settings +from scrapy.utils.asyncio import call_later +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.misc import build_from_crawler +from tests.utils.decorators import coroutine_test +from tests.utils.robotstxt import rerp_available + + +class TestRobotsTxtMiddleware: + def setup_method(self) -> None: + self.crawler: mock.MagicMock = mock.MagicMock() + self.crawler.settings = Settings() + self.crawler.engine.download_async = mock.AsyncMock() + self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[]) + + def teardown_method(self): + del self.crawler + + def test_robotstxt_settings(self): + self.crawler.settings = Settings() + self.crawler.settings.set("USER_AGENT", "CustomAgent") + with pytest.raises(NotConfigured): + build_from_crawler(RobotsTxtMiddleware, self.crawler) + + def _get_successful_crawler(self) -> mock.MagicMock: + crawler = self.crawler + crawler.settings.set("ROBOTSTXT_OBEY", True) + ROBOTS = """ +User-Agent: * +Disallow: /admin/ +Disallow: /static/ +# taken from https://en.wikipedia.org/robots.txt +Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4: +Disallow: /wiki/Käyttäjä: +User-Agent: UnicödeBöt +Disallow: /some/randome/page.html +""".encode() + response = TextResponse("http://site.local/robots.txt", body=ROBOTS) + + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() + call_later(0, deferred.callback, response) + return await maybe_deferred_to_future(deferred) + + crawler.engine.download_async.side_effect = return_response + return crawler + + @coroutine_test + async def test_robotstxt(self): + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_successful_crawler() + ) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + self.assertRobotsTxtRequested("http://site.local") + await self.assertIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertIgnored(Request("http://site.local/static/"), middleware) + await self.assertIgnored( + Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"), middleware + ) + await self.assertIgnored( + Request("http://site.local/wiki/Käyttäjä:"), middleware + ) + + @coroutine_test + async def test_robotstxt_emits_robots_parsed_signal(self): + crawler = self._get_successful_crawler() + middleware = RobotsTxtMiddleware(crawler) + request = Request("http://site.local/allowed") + await self.assertNotIgnored(request, middleware) + calls = [ + kwargs + for _, kwargs in crawler.signals.send_catch_log_async.call_args_list + if kwargs.get("signal") is signals.robots_parsed + ] + assert len(calls) == 1 + assert calls[0]["request"] is request + assert calls[0]["robotparser"] is not None + + @coroutine_test + async def test_robotstxt_multiple_reqs(self) -> None: + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_successful_crawler() + ) + d1 = deferred_from_coro( + middleware.process_request(Request("http://site.local/allowed1")) + ) + d2 = deferred_from_coro( + middleware.process_request(Request("http://site.local/allowed2")) + ) + await maybe_deferred_to_future(DeferredList([d1, d2], fireOnOneErrback=True)) + + @pytest.mark.only_asyncio + @coroutine_test + async def test_robotstxt_multiple_reqs_asyncio(self) -> None: + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_successful_crawler() + ) + c1 = middleware.process_request(Request("http://site.local/allowed1")) + c2 = middleware.process_request(Request("http://site.local/allowed2")) + await asyncio.gather(c1, c2) + + @coroutine_test + async def test_robotstxt_ready_parser(self): + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_successful_crawler() + ) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + + @coroutine_test + async def test_robotstxt_meta(self): + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_successful_crawler() + ) + meta = {"dont_obey_robotstxt": True} + await self.assertNotIgnored( + Request("http://site.local/allowed", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/admin/main", meta=meta), middleware + ) + await self.assertNotIgnored( + Request("http://site.local/static/", meta=meta), middleware + ) + + def _get_garbage_crawler(self) -> mock.MagicMock: + crawler = self.crawler + crawler.settings.set("ROBOTSTXT_OBEY", True) + response = Response( + "http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2" + ) + + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() + call_later(0, deferred.callback, response) + return await maybe_deferred_to_future(deferred) + + crawler.engine.download_async.side_effect = return_response + return crawler + + @coroutine_test + async def test_robotstxt_garbage(self): + # garbage response should be discarded, equal 'allow all' + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_garbage_crawler() + ) + await self.assertNotIgnored(Request("http://site.local"), middleware) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) + + def _get_emptybody_crawler(self) -> mock.MagicMock: + crawler = self.crawler + crawler.settings.set("ROBOTSTXT_OBEY", True) + response = Response("http://site.local/robots.txt") + + async def return_response(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() + call_later(0, deferred.callback, response) + return await maybe_deferred_to_future(deferred) + + crawler.engine.download_async.side_effect = return_response + return crawler + + @coroutine_test + async def test_robotstxt_empty_response(self): + # empty response should equal 'allow all' + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_emptybody_crawler() + ) + await self.assertNotIgnored(Request("http://site.local/allowed"), middleware) + await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware) + await self.assertNotIgnored(Request("http://site.local/static/"), middleware) + + @coroutine_test + async def test_robotstxt_error(self, caplog: pytest.LogCaptureFixture) -> None: + self.crawler.settings.set("ROBOTSTXT_OBEY", True) + err = CannotResolveHostError("Robotstxt address not found") + + async def return_failure(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() + call_later(0, deferred.errback, failure.Failure(err)) + return await maybe_deferred_to_future(deferred) + + self.crawler.engine.download_async.side_effect = return_failure + + middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler) + await middleware.process_request(Request("http://site.local")) + assert "Robotstxt address not found" in caplog.text + + @coroutine_test + async def test_robotstxt_immediate_error(self): + self.crawler.settings.set("ROBOTSTXT_OBEY", True) + err = CannotResolveHostError("Robotstxt address not found") + + async def immediate_failure(request): + raise err + + self.crawler.engine.download_async.side_effect = immediate_failure + + middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler) + await self.assertNotIgnored(Request("http://site.local"), middleware) + + @coroutine_test + async def test_ignore_robotstxt_request(self): + self.crawler.settings.set("ROBOTSTXT_OBEY", True) + + async def ignore_request(request: Request) -> Response: + deferred: Deferred[Response] = Deferred() + call_later(0, deferred.errback, failure.Failure(IgnoreRequest())) + return await maybe_deferred_to_future(deferred) + + self.crawler.engine.download_async.side_effect = ignore_request + + middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler) + with mock.patch( + "scrapy.downloadermiddlewares.robotstxt.logger" + ) as mw_module_logger: + await self.assertNotIgnored( + Request("http://site.local/allowed"), middleware + ) + assert not mw_module_logger.error.called + + def test_robotstxt_user_agent_setting(self): + crawler = self._get_successful_crawler() + crawler.settings.set("ROBOTSTXT_USER_AGENT", "Examplebot") + crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)") + middleware = build_from_crawler(RobotsTxtMiddleware, crawler) + rp = mock.MagicMock(return_value=True) + middleware.process_request_2(rp, Request("http://site.local/allowed")) + rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot") + + @coroutine_test + async def test_robotstxt_user_agent_none(self): + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_successful_crawler() + ) + headers = {"User-Agent": None} + await self.assertNotIgnored( + Request("http://site.local/allowed", headers=headers), middleware + ) + await self.assertIgnored( + Request("http://site.local/admin/main", headers=headers), middleware + ) + + @coroutine_test + async def test_robotstxt_local_file(self): + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_emptybody_crawler() + ) + middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign] + + await middleware.process_request(Request("data:text/plain,Hello World data")) + assert not middleware.process_request_2.called + + await middleware.process_request( + Request("file:///tests/sample_data/test_site/nothinghere.html") + ) + assert not middleware.process_request_2.called + + await middleware.process_request(Request("http://site.local/allowed")) + assert middleware.process_request_2.called + + @coroutine_test + async def test_robotstxt_download_handler(self) -> None: + middleware = RobotsTxtMiddleware(self._get_successful_crawler()) + await self.assertNotIgnored( + Request("http://site.local/allowed", meta={"download_handler": "named"}), + middleware, + ) + robotsreq = self.crawler.engine.download_async.call_args_list[0][0][0] + assert robotsreq.meta["download_handler"] == "named" + assert robotsreq.meta["is_robotstxt_request"] is True + + async def assertNotIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: + try: + await middleware.process_request(request) + except IgnoreRequest: + pytest.fail("IgnoreRequest was raised unexpectedly") + + async def assertIgnored( + self, request: Request, middleware: RobotsTxtMiddleware + ) -> None: + with pytest.raises(IgnoreRequest): + await middleware.process_request(request) + + def assertRobotsTxtRequested(self, base_url: str) -> None: + calls = self.crawler.engine.download_async.call_args_list + request = calls[0][0][0] + assert request.url == f"{base_url}/robots.txt" + assert request.callback == NO_CALLBACK + + +@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") +class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware): + def setup_method(self): + super().setup_method() + self.crawler.settings.set( + "ROBOTSTXT_PARSER", "scrapy.robotstxt.RerpRobotParser" + )