Celery Python Egg errors

[recrudesce]

Installed using setup.sh, throws this everytime Kraken starts

sudo /usr/bin/Kraken restart
Restarting Kraken.
[ ok ] Stopping rabbitmq-server (via systemctl): rabbitmq-server.service.
celery init v10.1.
Using config script: /etc/default/celeryd
Traceback (most recent call last):
  File "/usr/local/bin/celery", line 11, in <module>
    sys.exit(main())
  File "/usr/local/lib/python2.7/dist-packages/celery/__main__.py", line 13, in main
    from celery.bin.celery import main as _main
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/__init__.py", line 2, in <module>
    from .base import Option
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 17, in <module>
    from celery import VERSION_BANNER, Celery, maybe_patch_concurrency
  File "/usr/local/lib/python2.7/dist-packages/celery/local.py", line 509, in __getattr__
    module = __import__(self._object_origins[name], None, None, [name])
  File "/usr/local/lib/python2.7/dist-packages/celery/app/__init__.py", line 5, in <module>
    from celery import _state
  File "/usr/local/lib/python2.7/dist-packages/celery/_state.py", line 15, in <module>
    from celery.utils.threads import LocalStack
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/__init__.py", line 10, in <module>
    from .nodenames import worker_direct, nodename, nodesplit
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/nodenames.py", line 7, in <module>
    from kombu.entity import Exchange, Queue
  File "/usr/local/lib/python2.7/dist-packages/kombu/entity.py", line 9, in <module>
    from .serialization import prepare_accept_content
  File "/usr/local/lib/python2.7/dist-packages/kombu/serialization.py", line 396, in <module>
    register_msgpack()
  File "/usr/local/lib/python2.7/dist-packages/kombu/serialization.py", line 365, in register_msgpack
    import msgpack
  File "build/bdist.linux-x86_64/egg/msgpack/__init__.py", line 25, in <module>
  File "build/bdist.linux-x86_64/egg/msgpack/_packer.py", line 7, in <module>
  File "build/bdist.linux-x86_64/egg/msgpack/_packer.py", line 4, in __bootstrap__
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1172, in resource_filename
    self, resource_name
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1716, in get_resource_filename
    self._extract_resource(manager, self._eager_to_zip(name))
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1746, in _extract_resource
    self.egg_name, self._parts(zip_path)
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1239, in get_cache_path
    self.extraction_error()
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1219, in extraction_error
    raise err
pkg_resources.ExtractionError: Can't extract file(s) to egg cache

The following error occurred while trying to extract file(s) to the Python egg
cache:

  [Errno 13] Permission denied: '/home/celery'

The Python egg cache directory is currently set to:

  /home/celery/.python-eggs

Perhaps your account does not have write access to this directory?  You can
change the cache directory by setting the PYTHON_EGG_CACHE environment
variable to point to an accessible directory.

[ ok ] Stopping apache2 (via systemctl): apache2.service.
Kraken stopped.
[ ok ] Starting rabbitmq-server (via systemctl): rabbitmq-server.service.
celery init v10.1.
Using config script: /etc/default/celeryd
Traceback (most recent call last):
  File "/usr/local/bin/celery", line 11, in <module>
    sys.exit(main())
  File "/usr/local/lib/python2.7/dist-packages/celery/__main__.py", line 13, in main
    from celery.bin.celery import main as _main
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/__init__.py", line 2, in <module>
    from .base import Option
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 17, in <module>
    from celery import VERSION_BANNER, Celery, maybe_patch_concurrency
  File "/usr/local/lib/python2.7/dist-packages/celery/local.py", line 509, in __getattr__
    module = __import__(self._object_origins[name], None, None, [name])
  File "/usr/local/lib/python2.7/dist-packages/celery/app/__init__.py", line 5, in <module>
    from celery import _state
  File "/usr/local/lib/python2.7/dist-packages/celery/_state.py", line 15, in <module>
    from celery.utils.threads import LocalStack
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/__init__.py", line 10, in <module>
    from .nodenames import worker_direct, nodename, nodesplit
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/nodenames.py", line 7, in <module>
    from kombu.entity import Exchange, Queue
  File "/usr/local/lib/python2.7/dist-packages/kombu/entity.py", line 9, in <module>
    from .serialization import prepare_accept_content
  File "/usr/local/lib/python2.7/dist-packages/kombu/serialization.py", line 396, in <module>
    register_msgpack()
  File "/usr/local/lib/python2.7/dist-packages/kombu/serialization.py", line 365, in register_msgpack
    import msgpack
  File "build/bdist.linux-x86_64/egg/msgpack/__init__.py", line 25, in <module>
  File "build/bdist.linux-x86_64/egg/msgpack/_packer.py", line 7, in <module>
  File "build/bdist.linux-x86_64/egg/msgpack/_packer.py", line 4, in __bootstrap__
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1172, in resource_filename
    self, resource_name
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1716, in get_resource_filename
    self._extract_resource(manager, self._eager_to_zip(name))
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1746, in _extract_resource
    self.egg_name, self._parts(zip_path)
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1239, in get_cache_path
    self.extraction_error()
  File "/usr/lib/python2.7/dist-packages/pkg_resources/__init__.py", line 1219, in extraction_error
    raise err
pkg_resources.ExtractionError: Can't extract file(s) to egg cache

The following error occurred while trying to extract file(s) to the Python egg
cache:

  [Errno 13] Permission denied: '/home/celery'

The Python egg cache directory is currently set to:

  /home/celery/.python-eggs

Perhaps your account does not have write access to this directory?  You can
change the cache directory by setting the PYTHON_EGG_CACHE environment
variable to point to an accessible directory.

[ ok ] Starting apache2 (via systemctl): apache2.service.

Kraken started.
Open a browser and navigate to http://localhost:8000

Any thoughts ? I've set a PYTHON_EGG_CACHE environment variable, but still no joy. There is no /home/celery folder, and not sure what specifically requires it...

[recrudesce]

OK, fixed it by creating a home folder for celery, now the web interface throws a 403... I assume permissions related ?

[Sw4mpf0x]

Yeah, it definitely sounds like it. Can you confirm that a 'celery' user was added during setup? Also, do you have the setup output? Celery runs as the 'celery' user.

https://github.com/Sw4mpf0x/Kraken/blob/master/setup.sh#L52-L60

I have never had Celery try to access a /home/celery folder before so I am at a loss here. What Linux distro are you running this on?

[Sw4mpf0x]

If you are getting a 403 error, you should see the error output in /var/log/apache2/error.log. What gets logged when you try to access the web interface? That might help narrow in on the issue here. I have not used easy_install or Python eggs in this project, which is throwing me for a loop.

[recrudesce]

Celery user did get created, yeah. It fixed itself after I created the home folder. OS is Ubuntu 16.04 or 16.10 (can't remember specifically).

I'll get you Apache logs tomorrow. The issue might be cos I already had Apache installed (I think it overwrote my Apache config too, need to confirm that so don't take that as gospel).

[recrudesce]

[Tue Feb 07 12:12:19.843210 2017] [core:error] [pid 5989] (13)Permission denied: [client 127.0.0.1:51760] AH00035: access to / denied (filesystem path '/opt/Kraken/Kraken/wsgi.py') because search permissions are missing on a component of the path
[Tue Feb 07 12:12:19.991164 2017] [core:error] [pid 5989] (13)Permission denied: [client 127.0.0.1:51760] AH00035: access to /favicon.ico denied (filesystem path '/opt/Kraken/Kraken/wsgi.py') because search permissions are missing on a component of the path, referer: http://localhost:8000/

[Sw4mpf0x]

Looks like a permission issue with Apache. Try running these and restarting Apache/Kraken.

chown -R www-data /opt/Kraken chgrp -R www-data /opt/Kraken chmod 775 /opt/Kraken/static/Web_Scout chmod 775 /opt/Kraken/ghostdriver.log chmod 775 /opt/Kraken/tmp/ chmod 774 /opt/Kraken/Kraken/ chmod 774 /opt/Kraken/Kraken/kraken.db chown www-data /opt/Kraken/Kraken/kraken.db chgrp www-data /opt/Kraken/Kraken/kraken.db

The setup script does this, but maybe it got missed somehow.

[recrudesce]

OK, worked out what the problem is. I have apache already installed but it's not running as www-data. So the install script actually set the wrong permissions. Changed the permissions to be correct and now it runs, but throws this now

[ ok ] Starting rabbitmq-server (via systemctl): rabbitmq-server.service.
celery init v10.1.
Using config script: /etc/default/celeryd
celery multi v4.0.2 (latentcall)
> Starting nodes...
Traceback (most recent call last):
  File "/usr/lib/python2.7/runpy.py", line 174, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/usr/local/lib/python2.7/dist-packages/celery/__main__.py", line 18, in <module>
    main()
  File "/usr/local/lib/python2.7/dist-packages/celery/__main__.py", line 14, in main
    _main()
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 326, in main
    cmd.execute_from_commandline(argv)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/celery.py", line 488, in execute_from_commandline
    super(CeleryCommand, self).execute_from_commandline(argv)))
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 279, in execute_from_commandline
    argv = self.setup_app_from_commandline(argv)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 481, in setup_app_from_commandline
    self.app = self.find_app(app)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 503, in find_app
    return find_app(app, symbol_by_name=self.symbol_by_name)
  File "/usr/local/lib/python2.7/dist-packages/celery/app/utils.py", line 355, in find_app
    sym = symbol_by_name(app, imp=imp)
  File "/usr/local/lib/python2.7/dist-packages/celery/bin/base.py", line 506, in symbol_by_name
    return imports.symbol_by_name(name, imp=imp)
  File "/usr/local/lib/python2.7/dist-packages/kombu/utils/imports.py", line 56, in symbol_by_name
    module = imp(module_name, package=package, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/celery/utils/imports.py", line 101, in import_from_cwd
    return imp(module, package=package)
  File "/usr/lib/python2.7/importlib/__init__.py", line 37, in import_module
    __import__(name)
ImportError: No module named Kraken
    > krakenworker@senketsu: * Child terminated with exit code 1
FAILED
[ ok ] Starting apache2 (via systemctl): apache2.service.

Kraken started.
Open a browser and navigate to http://localhost:8000

The web interface works and allows me to log in, however everything stays at PENDING or I get ominous "failure" messages when adding URL's.

Getting there :)