Fix robots.txt not being applied to concurrent requests for a new host.

scrapy/downloadermiddlewares/robotstxt.py

@@ -113,7 +113,17 @@ class RobotsTxtMiddleware:
 
         parser = self._parsers[netloc]
         if isinstance(parser, Deferred):
-            return await maybe_deferred_to_future(parser)
+            # Awaiting a Deferred consumes its result, so every request waiting
+            # for this robots.txt gets its own Deferred, fired by a callback
+            # that also passes the result on to the next waiter.
+            waiter: Deferred[RobotParser | None] = Deferred()
+
+            def notify_waiter(rp: RobotParser | None) -> RobotParser | None:
+                waiter.callback(rp)
+                return rp
+
+            parser.addCallback(notify_waiter)
+            return await maybe_deferred_to_future(waiter)
         return parser
 
     async def _parse_robots(

tests/test_downloadermiddleware_robotstxt.py

@@ -113,6 +113,21 @@ Disallow: /some/randome/page.html
         c2 = middleware.process_request(Request("http://site.local/allowed2"))
         await asyncio.gather(c1, c2)
 
+    @coroutine_test
+    async def test_robotstxt_multiple_reqs_forbidden(self) -> None:
+        middleware = build_from_crawler(
+            RobotsTxtMiddleware, self._get_successful_crawler()
+        )
+        dfds = [
+            deferred_from_coro(
+                middleware.process_request(Request(f"http://site.local/admin/{i}"))
+            )
+            for i in range(3)
+        ]
+        for dfd in dfds:
+            with pytest.raises(IgnoreRequest):
+                await maybe_deferred_to_future(dfd)
+
     @coroutine_test
     async def test_robotstxt_ready_parser(self):
         middleware = build_from_crawler(