diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 210d525..1972de8 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -113,7 +113,17 @@ class RobotsTxtMiddleware: parser = self._parsers[netloc] if isinstance(parser, Deferred): - return await maybe_deferred_to_future(parser) + # Awaiting a Deferred consumes its result, so every request waiting + # for this robots.txt gets its own Deferred, fired by a callback + # that also passes the result on to the next waiter. + waiter: Deferred[RobotParser | None] = Deferred() + + def notify_waiter(rp: RobotParser | None) -> RobotParser | None: + waiter.callback(rp) + return rp + + parser.addCallback(notify_waiter) + return await maybe_deferred_to_future(waiter) return parser async def _parse_robots( diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 62b91e9..519444a 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -113,6 +113,21 @@ Disallow: /some/randome/page.html c2 = middleware.process_request(Request("http://site.local/allowed2")) await asyncio.gather(c1, c2) + @coroutine_test + async def test_robotstxt_multiple_reqs_forbidden(self) -> None: + middleware = build_from_crawler( + RobotsTxtMiddleware, self._get_successful_crawler() + ) + dfds = [ + deferred_from_coro( + middleware.process_request(Request(f"http://site.local/admin/{i}")) + ) + for i in range(3) + ] + for dfd in dfds: + with pytest.raises(IgnoreRequest): + await maybe_deferred_to_future(dfd) + @coroutine_test async def test_robotstxt_ready_parser(self): middleware = build_from_crawler(