search

TripleParity / docks-api

RESTful API Server for Docks
https://tripleparity.github.io/docks-api/stack-api-spec.html
GNU General Public License v3.0
0 stars 0 forks source link

Intermittent stack deploy errors #75

Closed egeldenhuys closed 6 years ago

egeldenhuys commented 6 years ago
Curl request to localhost
curl 'http://127.0.0.1:8080/stacks' -H 'User-Agent: Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:62.0) Gecko/20100101 Firefox/62.0' -H 'Accept: application/json, text/plain, */*' -H 'Accept-Language: en-AU,en-US;q=0.7,en;q=0.3' --compressed -H 'Referer: http://127.0.0.1:4200/stacks/create' -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VybmFtZSI6ImFkbWluIiwiaWF0IjoxNTM2OTMyOTMyfQ.ksJu8sQ9GD2dVheHN5h-TW3dFU6tp4lcQal4EVp71IY' -H 'Content-Type: application/json' -H 'Origin: http://127.0.0.1:4200' -H 'Connection: keep-alive' -H 'Pragma: no-cache' -H 'Cache-Control: no-cache' --data '{"stackName":"DASTACK","stackFile":"dmVyc2lvbjogJzMnCgpzZXJ2aWNlczoKICBtYWhTZXJ2aWNlOgogICAgaW1hZ2U6IG5naW54Cg=="}' -D -
Curl response from localhost
HTTP/1.1 500 Internal Server Error
X-Powered-By: Express
Access-Control-Allow-Origin: *
Content-Type: text/html; charset=utf-8
Content-Length: 36
ETag: W/"24-hIkj1S2rR4ZvUJpynEQrGMleEXY"
Date: Wed, 19 Sep 2018 16:09:20 GMT
Connection: keep-alive

Creating service DASTACK_mahService
Curl response (Changed url and tokens) from docks.evert.io
HTTP/2 200 
access-control-allow-origin: *
content-type: text/html; charset=utf-8
date: Wed, 19 Sep 2018 16:12:01 GMT
etag: W/"45-HZWxL3WnTJnv535ni9/w58zo2to"
x-powered-by: Express
content-length: 69

Creating network DASTACK_default
Creating service DASTACK_mahService
Decoded stack file
version: '3'

services:
  mahService:
    image: nginx
Docker version on localhost
$ docker -v
Docker version 18.06.1-ce, build e68fc7a
Docker version in container on localhost
$ docker -v
Docker version 18.06.0-ce, build 0ffa825
Stack file for deploying Docks API
version: '3.2'

services:

  api:
    image: tripleparity/docks-api:branch-sequelize-migrations-_72
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    environment:
      - JWT_SECRET=changeme
      - DOCKS_DB_ADDRESS=db
      - POSTGRES_PASSWORD=example
    ports:
      - '8080:8080'
    depends_on:
      - db
    deploy:
      placement:
        constraints: [node.role == manager]

  db:
    image: postgres
    volumes:
      - type: volume
        source: docks_data
        target: /var/lib/postgresql/data
    environment:
      - POSTGRES_PASSWORD=example

volumes:
  docks_data:
Docks API stack file on production
version: '3.2'

services:

  api:
    image: tripleparity/docks-api:branch-develop
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
    depends_on:
      - db
    networks:
      - default
      - traefik-net
    deploy:
      labels:
<snip>
      placement:
        constraints: [node.hostname == victor]
    environment:
      - JWT_SECRET=<snip>

  db:
    image: postgres
    networks:
      - default
    environment:
      - POSTGRES_PASSWORD=<snip>
    volumes:
      - type: volume
        source: docks_data
        target: /var/lib/postgresql/data

    deploy:
      placement:
        constraints: [node.hostname == victor]

networks:
  default:
  traefik-net:
    external: true

volumes:
  docks_data:
    driver: local
egeldenhuys commented 6 years ago

Problem: $ sudo dnf update Solution: $ reboot

egeldenhuys commented 6 years ago

Nope, after the reboot it worked once. But trying the same Curl request some time later results in:

HTTP/1.1 500 Internal Server Error
X-Powered-By: Express
Access-Control-Allow-Origin: *
Content-Type: text/html; charset=utf-8
Content-Length: 69
ETag: W/"45-HZWxL3WnTJnv535ni9/w58zo2to"
Date: Wed, 19 Sep 2018 16:45:45 GMT
Connection: keep-alive

Creating network DASTACK_default
Creating service DASTACK_mahService
egeldenhuys commented 6 years ago

After sudo systemctl restart docker stacks can be deployed using the Curl command with no problem. As soon as the stack is deployed from the UI, an error occurs and stacks can no longer be deployed using Curl.

Docks API Logs
Error deploying docker stack: Error: Command failed: docker stack deploy -c /tmp/1537375971079-64.yml STACK_FROM_UI

Preparing to deploy stack with name STACK_FROM_CURL

Error deploying docker stack: Error: Command failed: docker stack deploy -c /tmp/1537376071830-96.yml STACK_FROM_CURL
egeldenhuys commented 6 years ago

@devosray do you have any idea why the command is failing?

egeldenhuys commented 6 years ago
Docker events for a failed attempt
$ curl --no-buffer -XGET --unix-socket /var/run/docker.sock http:/v1.37/events

{"Type":"network","Action":"create","Actor":{"ID":"la9jj3u8kbfg9bsmhc0c7fq7l","Attributes":{"name":"THIS_STACK_HAS_FAILED_default"}},"scope":"swarm","time":1537378385,"timeNano":1537378385662707004}
{"Type":"network","Action":"update","Actor":{"ID":"la9jj3u8kbfg9bsmhc0c7fq7l","Attributes":{"name":"THIS_STACK_HAS_FAILED_default"}},"scope":"swarm","time":1537378385,"timeNano":1537378385694971893}
{"Type":"node","Action":"update","Actor":{"ID":"g6rd26vz1xc2p45ac9ruyq333","Attributes":{"name":"leonidas"}},"scope":"swarm","time":1537378385,"timeNano":1537378385731181868}
Good attempt
$ curl --no-buffer -XGET --unix-socket /var/run/docker.sock http:/v1.37/events

{"Type":"network","Action":"create","Actor":{"ID":"oabgbyx7z1saoptzneete7qke","Attributes":{"name":"STACK_GOOD_0_default"}},"scope":"swarm","time":1537378515,"timeNano":1537378515696081172}

{"Type":"network","Action":"update","Actor":{"ID":"oabgbyx7z1saoptzneete7qke","Attributes":{"name":"STACK_GOOD_0_default"}},"scope":"swarm","time":1537378515,"timeNano":1537378515713881539}

{"Type":"node","Action":"update","Actor":{"ID":"g6rd26vz1xc2p45ac9ruyq333","Attributes":{"name":"leonidas"}},"scope":"swarm","time":1537378515,"timeNano":1537378515724484413}

{"Type":"service","Action":"create","Actor":{"ID":"uy8spxiprgi3uihwkmowfhuuh","Attributes":{"name":"STACK_GOOD_0_mahService"}},"scope":"swarm","time":1537378520,"timeNano":1537378520385442386}

{"Type":"service","Action":"update","Actor":{"ID":"uy8spxiprgi3uihwkmowfhuuh","Attributes":{"name":"STACK_GOOD_0_mahService"}},"scope":"swarm","time":1537378520,"timeNano":1537378520412473780}

{"Type":"network","Action":"create","Actor":{"ID":"oabgbyx7z1saoptzneete7qke","Attributes":{"name":"STACK_GOOD_0_default","type":"overlay"}},"scope":"local","time":1537378521,"timeNano":1537378521619338138}

{"status":"create","id":"4f950a25fa80d448339a1b4d244e662f990b77fc34b089f46c886b61328ebf09","from":"nginx:latest@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3","Type":"container","Action":"create","Actor":{"ID":"4f950a25fa80d448339a1b4d244e662f990b77fc34b089f46c886b61328ebf09","Attributes":{"com.docker.stack.namespace":"STACK_GOOD_0","com.docker.swarm.node.id":"g6rd26vz1xc2p45ac9ruyq333","com.docker.swarm.service.id":"uy8spxiprgi3uihwkmowfhuuh","com.docker.swarm.service.name":"STACK_GOOD_0_mahService","com.docker.swarm.task":"","com.docker.swarm.task.id":"91fib0bbg01uv08rg1eicv3pg","com.docker.swarm.task.name":"STACK_GOOD_0_mahService.1.91fib0bbg01uv08rg1eicv3pg","image":"nginx:latest@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3","maintainer":"NGINX Docker Maintainers \u003cdocker-maint@nginx.com\u003e","name":"STACK_GOOD_0_mahService.1.91fib0bbg01uv08rg1eicv3pg"}},"scope":"local","time":1537378522,"timeNano":1537378522892833625}

{"Type":"network","Action":"connect","Actor":{"ID":"oabgbyx7z1saoptzneete7qke","Attributes":{"container":"4f950a25fa80d448339a1b4d244e662f990b77fc34b089f46c886b61328ebf09","name":"STACK_GOOD_0_default","type":"overlay"}},"scope":"local","time":1537378523,"timeNano":1537378523310455744}

{"status":"start","id":"4f950a25fa80d448339a1b4d244e662f990b77fc34b089f46c886b61328ebf09","from":"nginx:latest@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3","Type":"container","Action":"start","Actor":{"ID":"4f950a25fa80d448339a1b4d244e662f990b77fc34b089f46c886b61328ebf09","Attributes":{"com.docker.stack.namespace":"STACK_GOOD_0","com.docker.swarm.node.id":"g6rd26vz1xc2p45ac9ruyq333","com.docker.swarm.service.id":"uy8spxiprgi3uihwkmowfhuuh","com.docker.swarm.service.name":"STACK_GOOD_0_mahService","com.docker.swarm.task":"","com.docker.swarm.task.id":"91fib0bbg01uv08rg1eicv3pg","com.docker.swarm.task.name":"STACK_GOOD_0_mahService.1.91fib0bbg01uv08rg1eicv3pg","image":"nginx:latest@sha256:24a0c4b4a4c0eb97a1aabb8e29f18e917d05abfe1b7a7c07857230879ce7d3d3","maintainer":"NGINX Docker Maintainers \u003cdocker-maint@nginx.com\u003e","name":"STACK_GOOD_0_mahService.1.91fib0bbg01uv08rg1eicv3pg"}},"scope":"local","time":1537378524,"timeNano":1537378524097072146}
egeldenhuys commented 6 years ago

Temporary solution: Retry until it works