Unable to start a notebook....

[perllaghu]

I'm running Jupyterhub in a docker swarm, on an OpenStack base. I have 1 manager & 2 worker nodes.

I can happily create the 'Hub docker image, and log into it.... however that hub is not able to spawn notebooks.... and yes, the notebook has been pulled into the Swarm.

I'm getting the error: "json: cannot unmarshal string into Go value of type []string"

My docker file is:

FROM jupyterhub/jupyterhub
RUN apt-get update
RUN apt-get -y install python3-pip
# Temporary add these usful tools while developing
RUN apt-get -y install vim less net-tools
RUN useradd -ms /bin/bash test
RUN echo test:password | chpasswd
RUN git clone https://github.com/cassinyio/SwarmSpawner.git
WORKDIR SwarmSpawner
RUN pip install -r requirements.txt
RUN pip install .
WORKDIR /srv/jupyterhub
COPY config/jupyterhub_config.py /srv/jupyterhub/jupyterhub_config.py
CMD jupyterhub --debug > log.out 2>&1

and my config file is:

import subprocess
import os
import errno
import stat

c = get_config()
pwd = os.path.dirname(__file__)
c.JupyterHub.spawner_class = 'cassinyspawner.SwarmSpawner'
c.JupyterHub.ip = '0.0.0.0'
c.JupyterHub.hub_ip = '0.0.0.0'
c.JupyterHub.cleanup_servers = False
c.SwarmSpawner.start_timeout = 60 * 5
c.SwarmSpawner.jupyterhub_service_name = 'jupyterhub'
c.SwarmSpawner.networks = ["naas"]
notebook_dir = os.environ.get('NOTEBOOK_DIR') or '/home/test'
c.SwarmSpawner.notebook_dir = notebook_dir
mounts = [{'type' : 'volume',
           'source' : 'jupyterhub-user-{username}',
           'target' : notebook_dir}]
c.SwarmSpawner.container_spec = {
    'args' : '/usr/local/bin/start-singleuser.sh',
    'Image' : 'jupyterhub/singleuser',
    'mounts' : mounts
    }
c.SwarmSpawner.resource_spec = {
                'cpu_limit' : 1000,
                'mem_limit' : int(512 * 1e6),
                'cpu_reservation' : 1000,
                'mem_reservation' : int(512 * 1e6),
}

I start the hub with the following command: docker service create --constraint 'node.role==manager' --network naas --name jupyterhub -p 8000:8000 -p 8001:8001 -p 8081:8081 --env DOCKER_HOST=192.168.1.11:2375 naasswarm/jupyterhub (where 192.168.1.11 is the outward facing IP of the swarm)

When I log in, I get the following error-stack:

[D 2017-04-21 14:39:40.317 JupyterHub swarmspawner:221] Getting Docker service 'jupyter-098f6bcd4621d373cade4e832627b4f6-1'
[I 2017-04-21 14:39:40.321 JupyterHub swarmspawner:229] Docker service 'jupyter-098f6bcd4621d373cade4e832627b4f6-1' is gone
[W 2017-04-21 14:39:40.321 JupyterHub swarmspawner:194] Docker service not found
[E 2017-04-21 14:39:40.353 JupyterHub web:1548] Uncaught exception POST /hub/login?next= (10.255.0.2)
    HTTPServerRequest(protocol='http', host='172.16.48.141:8000', method='POST', uri='/hub/login?next=', version='HTTP/1.1', remote_ip='10.255.0.2', headers={'Content-Type': 'application/x-www-form-urlencoded', 'X-Forwarded-Host': '172.16.48.141:8000', 'Accept-Language': 'en-US,en;q=0.5', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:52.0) Gecko/20100101 Firefox/52.0', 'Cookie': '_ga=GA1.1.103564032.1490776825; toggle=true', 'Upgrade-Insecure-Requests': '1', 'X-Forwarded-Proto': 'http', 'Content-Length': '31', 'Referer': 'http://172.16.48.141:8000/hub/login', 'X-Forwarded-Port': '8000', 'Connection': 'close', 'Accept-Encoding': 'gzip, deflate', 'Host': '172.16.48.141:8000', 'X-Forwarded-For': '10.255.0.2', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'})
    Traceback (most recent call last):
      File "/opt/conda/lib/python3.5/site-packages/tornado/web.py", line 1469, in _execute
        result = yield result
      File "/opt/conda/lib/python3.5/site-packages/jupyterhub/handlers/login.py", line 83, in post
        yield self.spawn_single_user(user)
      File "/opt/conda/lib/python3.5/site-packages/jupyterhub/handlers/base.py", line 325, in spawn_single_user
        yield gen.with_timeout(timedelta(seconds=self.slow_spawn_timeout), f)
      File "/opt/conda/lib/python3.5/site-packages/jupyterhub/user.py", line 310, in spawn
        raise e
      File "/opt/conda/lib/python3.5/site-packages/jupyterhub/user.py", line 278, in spawn
        ip_port = yield gen.with_timeout(timedelta(seconds=spawner.start_timeout), f)
      File "/opt/conda/lib/python3.5/site-packages/cassinyspawner/swarmspawner.py", line 311, in start
        networks=networks)
      File "/opt/conda/lib/python3.5/concurrent/futures/_base.py", line 398, in result
        return self.__get_result()
      File "/opt/conda/lib/python3.5/concurrent/futures/_base.py", line 357, in __get_result
        raise self._exception
      File "/opt/conda/lib/python3.5/concurrent/futures/thread.py", line 55, in run
        result = self.fn(*self.args, **self.kwargs)
      File "/opt/conda/lib/python3.5/site-packages/cassinyspawner/swarmspawner.py", line 180, in _docker
        return m(*args, **kwargs)
      File "/opt/conda/lib/python3.5/site-packages/docker/utils/decorators.py", line 35, in wrapper
        return f(self, *args, **kwargs)
      File "/opt/conda/lib/python3.5/site-packages/docker/api/service.py", line 84, in create_service
        self._post_json(url, data=data, headers=headers), True
      File "/opt/conda/lib/python3.5/site-packages/docker/api/client.py", line 220, in _result
        self._raise_for_status(response)
      File "/opt/conda/lib/python3.5/site-packages/docker/api/client.py", line 216, in _raise_for_status
        raise create_api_error_from_http_exception(e)
      File "/opt/conda/lib/python3.5/site-packages/docker/errors.py", line 30, in create_api_error_from_http_exception
        raise cls(e, response=response, explanation=explanation)
    docker.errors.APIError: 500 Server Error: Internal Server Error ("json: cannot unmarshal string into Go value of type []string")

(and by dint of a bit of hacking, I can confirm that the

resp = yield self.docker('create_service',
                                     task_tmpl,
                                     name=self.service_name,
                                     networks=networks)

line is being fed the following data:

[I 2017-04-21 14:39:40.312 JupyterHub swarmspawner:307] Call Docker create_service task_template: {'ContainerSpec': {'Args': '/usr/local/bin/start-singleuser.sh',
                       'Command': None,
                       'Env': ['JPY_BASE_URL=/user/test',
                               'JPY_USER=test',
                               'JPY_COOKIE_NAME=jupyter-hub-token-test',
                               'JUPYTERHUB_CLIENT_ID=user-test',
                               'JPY_HUB_PREFIX=/hub/',
                               'JUPYTERHUB_API_TOKEN=e80c8e73010349bb85d846958b667bca',
                               'JUPYTERHUB_HOST=',
                               'NOTEBOOK_DIR=/home/test',
                               'JUPYTERHUB_OAUTH_CALLBACK_URL=/user/test/oauth_callback',
                               'JPY_API_TOKEN=e80c8e73010349bb85d846958b667bca',
                               'JPY_HUB_API_URL=http://jupyterhub:8081/hub/api'],
                       'Image': 'jupyterhub/singleuser',
                       'Mounts': [{'ReadOnly': False,
                                   'Source': 'jupyterhub-user-098f6bcd4621d373cade4e832627b4f6',
                                   'Target': '/home/test',
                                   'Type': 'volume'}]},
     'Resources': {'Limits': {'MemoryBytes': 512000000, 'NanoCPUs': 1000},
                   'Reservations': {'MemoryBytes': 512000000, 'NanoCPUs': 1000}}},
     name:'jupyter-098f6bcd4621d373cade4e832627b4f6-1',
     networks: ['naas']

)

I'm at a loss where the []string is, and actually what's tripping up....

Help?

[barrachri]

I guess it is a change in the python api for Docker that now expects something different....

Let me give a look.

In the meantime thanks for opening the issue :)

[barrachri]

which version of Docker are you using?

[perllaghu]

I'll come back to you in about an hour.... I'm away from my computer just now....

[barrachri]

I think I got the error:

args should be a list

'args' : ['/usr/local/bin/start-singleuser.sh']

instead of a string

'args' : '/usr/local/bin/start-singleuser.sh'

I am going to update the example.

[perllaghu]

Getting there - I can actually get the Hub to start services...

There seems to be a requirement for notebook >= 5 .... so doing some updating

More updates as I get through it

[perllaghu]

Update: I can make it work.... but I have to:

1. Start the hub docker
2. Manually start a jupyter/base-notebook` image (in the swarm)
3. Then I can log in & start a different Notebook (I've chosen a jupyter/datascience-notebook

I need to play some more to see what's the minimum mucking-about I need .... and still have things work.

[barrachri]

Why you have to do the point n. 2?

[perllaghu]

I don't know.... I did it & I got a notebook for the first time :D :D

I've just tried removing & rebuilding the hub, but leaving the base-notebook in the swarm, and it's failed... I need to try some more stuff

[barrachri]

Ok, consider that the docker/jupyter images are quite big...

[perllaghu]

Aye.... I'm seeing that. (my step after getting this going, is to port DockerImageChooserSpawner across to sub-class SwarmSpawner rather than DockerSpawner)

[barrachri]

You can already pick a different image using the user_options.

[perllaghu]

OK.... I think I've cracked it... the problem is with jupyterhub/singleuser - if I add c.SwarmSpawner.service_image = 'jupyter/base-notebook' to my config file, and call 'Image' : 'jupyter/datascience-notebook', we're good.

(I'm using Docker 17.04 on my workstation, and 17.03 on the Swarm service)

I'm just going to do a massive docker-image wipe, and rebuild from scratch... just to be sure!

[perllaghu]

This is my working configuration

Dockerfile

# Needs to be 0.7.2 'cos latest doesn't work
FROM jupyterhub/jupyterhub:0.7.2

RUN apt-get update
RUN apt-get -y install python3-pip
# Temporary add these usful tools while developing
RUN apt-get -y install vim less net-tools
RUN useradd -ms /bin/bash test
RUN echo test:password | chpasswd
RUN git clone https://github.com/cassinyio/SwarmSpawner.git
WORKDIR SwarmSpawner
RUN pip install -r requirements.txt
RUN pip install .
WORKDIR /srv/jupyterhub
COPY config/jupyterhub_config.py /srv/jupyterhub/jupyterhub_config.py
CMD jupyterhub --debug

config/jupyterhub_config.py

import subprocess
import os
import errno
import stat

c = get_config()
pwd = os.path.dirname(__file__)

c.JupyterHub.spawner_class = 'cassinyspawner.SwarmSpawner'

c.JupyterHub.ip = '0.0.0.0'
c.JupyterHub.hub_ip = '0.0.0.0'

c.JupyterHub.cleanup_servers = False

# First pulls can be really slow, so let's give it a big timeout
c.SwarmSpawner.start_timeout = 60 * 5

c.SwarmSpawner.jupyterhub_service_name = 'jupyterhub'

c.SwarmSpawner.networks = ["naas"]

notebook_dir = os.environ.get('NOTEBOOK_DIR') or '/home/test'
c.SwarmSpawner.notebook_dir = notebook_dir

# Need to over-ride the default in SwarmSpawner, as that's Notebook 4
# Image needs to be previously pulled
c.SwarmSpawner.service_image = 'jupyter/base-notebook'

mounts = [{'type' : 'volume',
           'source' : 'jupyterhub-user-{username}',
           'target' : notebook_dir}]

# 'args' is the command to run inside the service
c.SwarmSpawner.container_spec = {
    'args' : ['/usr/local/bin/start-singleuser.sh'],
    # image needs to be previously pulled
    'Image' : 'jupyter/datascience-notebook',
    'mounts' : mounts,
    }

# The values here are too low for our OpenStack system
c.SwarmSpawner.resource_spec = {
#                'cpu_limit' : 1000000,
#                'mem_limit' : int(512 * 1e6),
#                'cpu_reservation' : 1000000,
#                'mem_reservation' : int(512 * 1e6),
}

Thank you for your help!