SahDoum commented 4 weeks ago

Fetcher redesign

Vldln commented 4 weeks ago

>>> 
now exiting InteractiveConsole...
^[[6~root@62ca4ed51694:/code# python manage.py new_inc
ERROR:server.apps.core.management.commands.new_inc:Failed to create an incident: 'NoneType' object has no attribute 'description'

@SahDoum

Vldln commented 4 weeks ago

from server.celery.crawler import crawl_chain crawl_chain.apply()

Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/celery/app/trace.py", line 453, in trace_task
R = retval = fun(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/sentry_sdk/integrations/celery.py", line 200, in _inner
reraise(*exc_info)
File "/usr/local/lib/python3.9/site-packages/sentry_sdk/_compat.py", line 54, in reraise
raise value
File "/usr/local/lib/python3.9/site-packages/sentry_sdk/integrations/celery.py", line 195, in _inner
return f(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/celery/app/trace.py", line 736, in __protected_call__
return self.run(*args, **kwargs)
File "/code/server/celery/crawler.py", line 35, in fetch_sources
fetched_count = fetcher.await_tasks()
File "/code/server/core/fetcher/fetcher.py", line 69, in await_tasks
results = asyncio.run(self._await())
File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete
return future.result()
File "/code/server/core/fetcher/fetcher.py", line 65, in _await
results = await asyncio.gather(*self.coroutines, return_exceptions=False)
File "/code/server/core/fetcher/tasks.py", line 42, in fetch_source_articles
return statistics._postprocess
AttributeError: 'CoroutineStatistics' object has no attribute '_postprocess'

@SahDoum

SahDoum commented 4 weeks ago

>>> 
now exiting InteractiveConsole...
^[[6~root@62ca4ed51694:/code# python manage.py new_inc
ERROR:server.apps.core.management.commands.new_inc:Failed to create an incident: 'NoneType' object has no attribute 'description'

@SahDoum

python manage.py new_inc

DEBUG:asyncio:Using selector: EpollSelector INFO:server.apps.core.management.commands.new_inc:Successfully created and saved an incident: [Sample Article Title]

SahDoum commented 4 weeks ago

from server.celery.crawler import crawl_chain crawl_chain.apply()

Traceback (most recent call last):
  File "/usr/local/lib/python3.9/site-packages/celery/app/trace.py", line 453, in trace_task
    R = retval = fun(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/sentry_sdk/integrations/celery.py", line 200, in _inner
    reraise(*exc_info)
  File "/usr/local/lib/python3.9/site-packages/sentry_sdk/_compat.py", line 54, in reraise
    raise value
  File "/usr/local/lib/python3.9/site-packages/sentry_sdk/integrations/celery.py", line 195, in _inner
    return f(*args, **kwargs)
  File "/usr/local/lib/python3.9/site-packages/celery/app/trace.py", line 736, in __protected_call__
    return self.run(*args, **kwargs)
  File "/code/server/celery/crawler.py", line 35, in fetch_sources
    fetched_count = fetcher.await_tasks()
  File "/code/server/core/fetcher/fetcher.py", line 69, in await_tasks
    results = asyncio.run(self._await())
  File "/usr/local/lib/python3.9/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/usr/local/lib/python3.9/asyncio/base_events.py", line 647, in run_until_complete
    return future.result()
  File "/code/server/core/fetcher/fetcher.py", line 65, in _await
    results = await asyncio.gather(*self.coroutines, return_exceptions=False)
  File "/code/server/core/fetcher/tasks.py", line 42, in fetch_source_articles
    return statistics._postprocess
AttributeError: 'CoroutineStatistics' object has no attribute '_postprocess'

@SahDoum

Поправил

antijob / neuro-parser

Feature/fetcher #212

python manage.py new_inc