search

iw4p / proxy-scraper

scrape proxies from more than 5 different sources and check which ones are still alive
MIT License
514 stars 131 forks source link

Unable to run the proxy scraper #26

Closed BukuBukuChagma closed 1 year ago

BukuBukuChagma commented 1 year ago

When I run it after installing all the requirements, this is what I get

E:\scripts\proxy-scraper>python proxyScraper.py -p http Traceback (most recent call last): File "E:\scripts\proxy-scraper\proxyScraper.py", line 169, in asyncio.run(scrape(args.proxy, args.output, args.verbose)) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\runners.py", line 44, in run return loop.run_until_complete(main) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 646, in run_until_complete return future.result() File "E:\scripts\proxy-scraper\proxyScraper.py", line 136, in scrape await asyncio.gather(*tasks) File "E:\scripts\proxy-scraper\proxyScraper.py", line 131, in scrape_scraper proxies.extend(await scraper.scrape(client)) File "E:\scripts\proxy-scraper\proxyScraper.py", line 27, in scrape response = await self.get_response(client) File "E:\scripts\proxy-scraper\proxyScraper.py", line 21, in get_response return await client.get(self.get_url()) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1757, in get return await self.request( File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1533, in request return await self.send(request, auth=auth, follow_redirects=follow_redirects) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1620, in send response = await self._send_handling_auth( File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1648, in _send_handling_auth response = await self._send_handling_redirects( File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1685, in _send_handling_redirects response = await self._send_single_request(request) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1722, in _send_single_request response = await transport.handle_async_request(request) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_transports\default.py", line 353, in handle_async_request resp = await self._pool.handle_async_request(req) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection_pool.py", line 253, in handle_async_request raise exc File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection_pool.py", line 237, in handle_async_request response = await connection.handle_async_request(request) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 86, in handle_async_request raise exc File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 63, in handle_async_request stream = await self._connect(request) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 150, in _connect stream = await stream.start_tls(*kwargs) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore\backends\asyncio.py", line 76, in start_tls raise exc File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore\backends\asyncio.py", line 67, in start_tls ssl_stream = await anyio.streams.tls.TLSStream.wrap( File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\anyio\streams\tls.py", line 122, in wrap await wrapper._call_sslobject_method(ssl_object.do_handshake) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\anyio\streams\tls.py", line 130, in _call_sslobject_method result = func(args) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 975, in do_handshake self._sslobj.do_handshake() ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:997) Exception ignored in: <function _ProactorBasePipeTransport.del at 0x000001813AE9E3B0> Traceback (most recent call last): File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 116, in del File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 108, in close File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 750, in call_soon File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 515, in _check_closed RuntimeError: Event loop is closed Exception ignored in: <function _ProactorBasePipeTransport.del at 0x000001813AE9E3B0> Traceback (most recent call last): File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 116, in del File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 108, in close File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 750, in call_soon File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 515, in _check_closed RuntimeError: Event loop is closed Exception ignored in: <function _ProactorBasePipeTransport.del at 0x000001813AE9E3B0> Traceback (most recent call last): File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 116, in del File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 108, in close File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 750, in call_soon File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 515, in _check_closed RuntimeError: Event loop is closed Exception ignored in: <function _ProactorBasePipeTransport.del at 0x000001813AE9E3B0> Traceback (most recent call last): File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 116, in del File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\proactor_events.py", line 108, in close File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 750, in call_soon File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 515, in _check_closed RuntimeError: Event loop is closed

E:\scripts\proxy-scraper>

Am I doing something wrong here?

iw4p commented 1 year ago

Hi, I updated the source code. Please get the new source code and test it again. If you still have a problem, let me know; Because I don't have access to windows to test it.

BukuBukuChagma commented 1 year ago

Cloned the updated repo but looks like the error is still there. Exact same error as the one above ssl handshake error.

iw4p commented 1 year ago

Sorry, my bad. I updated it again. Please try again. If it does not work again, try to remove lines from 169 to 179 and just keep:

        loop = asyncio.get_event_loop()
        loop.run_until_complete(scrape(args.proxy, args.output, args.verbose))
        loop.close()

You can read more about the problem here.

BukuBukuChagma commented 1 year ago

The event loop problem is gone but this error still exists:

ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:997)

I removed all the lines from 169 onward and kept the ones you mentioned and it looks like this line is causing the error:

loop = asyncio.get_event_loop()
loop.run_until_complete(scrape(args.proxy, args.output, args.verbose))

Complete error:

E:\scripts\proxy-scraper>python proxyScraper.py -p http E:\scripts\proxy-scraper\proxyScraper.py:170: DeprecationWarning: There is no current event loop loop = asyncio.get_event_loop() Traceback (most recent call last): File "E:\scripts\proxy-scraper\proxyScraper.py", line 171, in loop.run_until_complete(scrape(args.proxy, args.output, args.verbose)) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\asyncio\base_events.py", line 646, in run_until_complete return future.result() File "E:\scripts\proxy-scraper\proxyScraper.py", line 137, in scrape await asyncio.gather(*tasks) File "E:\scripts\proxy-scraper\proxyScraper.py", line 132, in scrape_scraper proxies.extend(await scraper.scrape(client)) File "E:\scripts\proxy-scraper\proxyScraper.py", line 28, in scrape response = await self.get_response(client) File "E:\scripts\proxy-scraper\proxyScraper.py", line 22, in get_response return await client.get(self.get_url()) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1757, in get return await self.request( File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1533, in request return await self.send(request, auth=auth, follow_redirects=follow_redirects) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1620, in send response = await self._send_handling_auth( File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1648, in _send_handling_auth response = await self._send_handling_redirects( File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1685, in _send_handling_redirects response = await self._send_single_request(request) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_client.py", line 1722, in _send_single_request response = await transport.handle_async_request(request) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpx_transports\default.py", line 353, in handle_async_request resp = await self._pool.handle_async_request(req) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection_pool.py", line 253, in handle_async_request raise exc File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection_pool.py", line 237, in handle_async_request response = await connection.handle_async_request(request) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 86, in handle_async_request raise exc File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 63, in handle_async_request stream = await self._connect(request) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore_async\connection.py", line 150, in _connect stream = await stream.start_tls(*kwargs) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore\backends\asyncio.py", line 76, in start_tls raise exc File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\httpcore\backends\asyncio.py", line 67, in start_tls ssl_stream = await anyio.streams.tls.TLSStream.wrap( File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\anyio\streams\tls.py", line 122, in wrap await wrapper._call_sslobject_method(ssl_object.do_handshake) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\site-packages\anyio\streams\tls.py", line 130, in _call_sslobject_method result = func(args) File "C:\Users\choud\AppData\Local\Programs\Python\Python310\lib\ssl.py", line 975, in do_handshake self._sslobj.do_handshake() ssl.SSLError: [SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:997)