This is one of those situations where you get capped for downloading too much shit at once. Setting 1 worker yields the same result.
Traceback (most recent call last):
File "C:\tools\python\Lib\site-packages\aiohttp\connector.py", line 1025, in _wrap_create_connection
return await self._loop.create_connection(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\asyncio\base_events.py", line 1122, in create_connection
raise exceptions[0]
File "C:\tools\python\Lib\asyncio\base_events.py", line 1104, in create_connection
sock = await self._connect_sock(
^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\asyncio\base_events.py", line 1007, in _connect_sock
await self.sock_connect(sock, address)
File "C:\tools\python\Lib\asyncio\proactor_events.py", line 727, in sock_connect
return await self._proactor.connect(sock, address)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\asyncio\windows_events.py", line 803, in _poll
value = callback(transferred, key, ov)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\asyncio\windows_events.py", line 599, in finish_connect
ov.getresult()
ConnectionRefusedError: [WinError 1225] The remote computer refused the network connection
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\dev\dl_dirlisting.py", line 18, in <module>
asyncio.run(main())
File "C:\tools\python\Lib\asyncio\runners.py", line 194, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\asyncio\runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\asyncio\base_events.py", line 687, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "C:\dev\dl_dirlisting.py", line 14, in main
await downloader.crawl(url)
File "C:\tools\python\Lib\site-packages\directory_downloader\directory_downloader.py", line 69, in crawl
await self.crawl(link)
File "C:\tools\python\Lib\site-packages\directory_downloader\directory_downloader.py", line 69, in crawl
await self.crawl(link)
File "C:\tools\python\Lib\site-packages\directory_downloader\directory_downloader.py", line 69, in crawl
await self.crawl(link)
[Previous line repeated 17 more times]
File "C:\tools\python\Lib\site-packages\directory_downloader\directory_downloader.py", line 65, in crawl
links = await self.get_page_links(url)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\site-packages\directory_downloader\directory_downloader.py", line 35, in get_page_links
response = await self._get_source_code(url)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\site-packages\directory_downloader\directory_downloader.py", line 78, in _get_source_code
async with session.get(url) as resp:
File "C:\tools\python\Lib\site-packages\aiohttp\client.py", line 1197, in __aenter__
self._resp = await self._coro
^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\site-packages\aiohttp\client.py", line 581, in _request
conn = await self._connector.connect(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\site-packages\aiohttp\connector.py", line 544, in connect
proto = await self._create_connection(req, traces, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\site-packages\aiohttp\connector.py", line 944, in _create_connection
_, proto = await self._create_direct_connection(req, traces, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\site-packages\aiohttp\connector.py", line 1257, in _create_direct_connection
raise last_exc
File "C:\tools\python\Lib\site-packages\aiohttp\connector.py", line 1226, in _create_direct_connection
transp, proto = await self._wrap_create_connection(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\tools\python\Lib\site-packages\aiohttp\connector.py", line 1033, in _wrap_create_connection
raise client_error(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host web.archive.org:443 ssl:default [The remote computer refused the network connection]
This is one of those situations where you get capped for downloading too much shit at once. Setting 1 worker yields the same result.