import asyncio
from .log import logger
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
except ImportError:
pass
async def fetch(url, spider, session, semaphore):
with (await semaphore):
try:
if callable(spider.headers):
headers = spider.headers()
else:
headers = spider.headers
# hare hare hare
if hasattr(spider,'encoding'):
codec = spider.encoding
else:
codec = 'utf-8'
# hare hare hare
async with session.get(url, headers=headers) as response:
if response.status in [200, 201]:
data = await response.text(encoding=codec) # hare hare hare
return data
logger.error('Error: {} {}'.format(url, response.status))
return None
except:
return None
request.py
test.py