update docker-compose - Githubissues

konvergence commented 4 years ago

Hi,

Here a proposal of docker-compose.yml that not use host path for /var/run and not use volumes_from

version: '2.1'
volumes:
  redis-data: {}
  openvas-var-lib: {}
  gvm-var-lib: {}
  postgres-data: {}
  run-redis: {}
  run-ospd: {}

services:
  gvm-postgres:
    image: admirito/gvm-postgres:11
    environment:
      PGDATA: /var/lib/postgresql/data
      POSTGRES_DB: gvmd
      POSTGRES_PASSWORD: mypassword
      POSTGRES_USER: gvmduser
    stdin_open: true
    volumes:
    - postgres-data:/var/lib/postgresql/data

  gvmd:
    #  CONNECTED     9598310  /var/run/ospd/ospd.sock
    image: admirito/gvmd:11
    environment:
      GVMD_POSTGRESQL_URI: postgresql://gvmduser:mypassword@gvm-postgres:5432/gvmd?application_name=gvmd
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - gvm-var-lib:/var/lib/gvm
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    depends_on:
      postgres:
        condition: gvm-postgres

  gsad:
    image: admirito/gsad:11
    ports:
    - 8080:80

    environment:
      GVMD_HOST: gvmd
      GVMD_PORT: '9390'

    depends_on:
      postgres:
        condition: gvmd

  openvas:
    # LISTENING     9431657  /var/run/ospd/ospd.sock
    # CONNECTED     9499517  /var/run/redis/redis.sock
    image: admirito/openvas:11
    environment:
      OV_PASSWORD: Securepassword41
    privileged: true
    sysctls:
      net.core.somaxconn: '2048'
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    depends_on:
      postgres:
        condition: gvmd

# on node must  add     vm.overcommit_memory=1 into /etc/systcl.conf
  redis:
    # LISTENING     9418817  /var/run/redis/redis.sock
    image: redis:5.0
    volumes:
    - run-redis:/var/run/redis
    - redis-data:/data
    command: redis-server --port 0 --unixsocket /var/run/redis/redis.sock --unixsocketperm 755
    privileged: true
    sysctls:
      net.core.somaxconn: '2048'
    depends_on:
      postgres:
        condition: openvas

  cert-sync:
    image: admirito/gvmd:11
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - gvm-var-lib:/var/lib/gvm
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    command: greenbone-certdata-sync --curl --verbose

    depends_on:
      postgres:
        condition: gvmd

  scap-sync:
    image: admirito/gvmd:11
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - gvm-var-lib:/var/lib/gvm
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    command: greenbone-scapdata-sync --curl --verbose

    depends_on:
      postgres:
        condition: gvmd

  nvt-sync:
    image: admirito/openvas:11
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    command: greenbone-nvt-sync

    depends_on:
      postgres:
        condition: gvmd

konvergence commented 4 years ago

please look my pull request

avancerapide commented 4 years ago

Hello @konvergence,

I try your proposal of docker-compose.yml but when I launch "docker-compose up" it return me these messages:

ERROR: The Compose file './docker-compose.yml' is invalid because:
services.gvmd.depends_on.postgres.condition contains "gvm-postgres", which is an invalid type, it should be a service_started, or a service_healthy
services.cert-sync.depends_on.postgres.condition contains "gvmd", which is an invalid type, it should be a service_started, or a service_healthy
services.gsad.depends_on.postgres.condition contains "gvmd", which is an invalid type, it should be a service_started, or a service_healthy
services.nvt-sync.depends_on.postgres.condition contains "gvmd", which is an invalid type, it should be a service_started, or a service_healthy
services.openvas.depends_on.postgres.condition contains "gvmd", which is an invalid type, it should be a service_started, or a service_healthy
services.scap-sync.depends_on.postgres.condition contains "gvmd", which is an invalid type, it should be a service_started, or a service_healthy
services.redis.depends_on.postgres.condition contains "openvas", which is an invalid type, it should be a service_started, or a service_healthy

konvergence commented 4 years ago

Yes sorry

Here the good one

version: '2.1'
volumes:
  redis-data: {}
  openvas-var-lib: {}
  gvm-var-lib: {}
  postgres-data: {}
  run-redis: {}
  run-ospd: {}

services:
  gvm-postgres:
    image: admirito/gvm-postgres:11
    environment:
      PGDATA: /var/lib/postgresql/data
      POSTGRES_DB: gvmd
      POSTGRES_PASSWORD: mypassword
      POSTGRES_USER: gvmduser
    stdin_open: true
    volumes:
    - postgres-data:/var/lib/postgresql/data

  gvmd:
    #  CONNECTED     9598310  /var/run/ospd/ospd.sock
    image: admirito/gvmd:11
    environment:
      GVMD_POSTGRESQL_URI: postgresql://gvmduser:mypassword@gvm-postgres:5432/gvmd?application_name=gvmd
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - gvm-var-lib:/var/lib/gvm
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    depends_on:
      gvm-postgres:
        condition: service_started

  gsad:
    image: admirito/gsad:11
    ports:
    - 8080:80

    environment:
      GVMD_HOST: gvmd
      GVMD_PORT: '9390'

    depends_on:
      gvmd:
        condition: service_started

  openvas:
    # LISTENING     9431657  /var/run/ospd/ospd.sock
    # CONNECTED     9499517  /var/run/redis/redis.sock
    image: admirito/openvas:11
    environment:
      OV_PASSWORD: Securepassword41
    privileged: true
    sysctls:
      net.core.somaxconn: '2048'
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd

    depends_on:
      gvm-postgres:
        condition: service_started

# on node must  add     vm.overcommit_memory=1 into /etc/systcl.conf
  redis:
    # LISTENING     9418817  /var/run/redis/redis.sock
    image: redis:5.0
    volumes:
    - run-redis:/var/run/redis
    - redis-data:/data
    command: redis-server --port 0 --unixsocket /var/run/redis/redis.sock --unixsocketperm 755
    privileged: true
    sysctls:
      net.core.somaxconn: '2048'

    depends_on:
      openvas:
        condition: service_started

  cert-sync:
    image: admirito/gvmd:11
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - gvm-var-lib:/var/lib/gvm
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    command: greenbone-certdata-sync --curl --verbose

    depends_on:
      gvmd:
        condition: service_started

  scap-sync:
    image: admirito/gvmd:11
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - gvm-var-lib:/var/lib/gvm
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    command: greenbone-scapdata-sync --curl --verbose

    depends_on:
      gvmd:
        condition: service_started

  nvt-sync:
    image: admirito/openvas:11
    volumes:
    - openvas-var-lib:/var/lib/openvas
    - run-redis:/var/run/redis
    - run-ospd:/var/run/ospd
    command: greenbone-nvt-sync

    depends_on:
      gvmd:
        condition: service_started

avancerapide commented 4 years ago

Thanks, I try it, docker-compose up running, but I've many other warnings messages like:

redis_1         | 1:M 27 Mar 2020 10:19:04.976 # You requested maxclients of 10000 requiring at least 10032 max file descriptors.
redis_1         | 1:M 27 Mar 2020 10:19:04.976 # Server can't set maximum open files to 10032 because of OS error: Operation not permitted.

and

gvmd_1 | md manage:WARNING:2020-03-27 09h52.51 utc:171: manage_update_nvt_cache_osp: failed to connect to /var/run/ospd/ospd.sock

I will investigate ... :-)

konvergence commented 4 years ago

this messages are because you have to change the max file descriptors https://www.cyberciti.biz/faq/linux-increase-the-maximum-number-of-open-files/

or update your docker-compose with

ulimits:
    nofile:
       soft: 65536
       hard: 65536

avancerapide commented 4 years ago

Thanks a lot, ulimit is ok for redis, but the warning in gvmd is still here and I can't connect to the web app

konvergence commented 4 years ago

did you use my latest docker-compose.yml (with the ulimits update for resdis) ? Because it fork fine for me

avancerapide commented 4 years ago

Yes I test with ulimits this correct the limit message but not the second message on redis: redis_1 | 1:M 27 Mar 2020 11:31:25.299 # Opening Unix socket: bind: Permission denied and after a while gvm-containers_redis_1 exited with code 1

to correct this messages, I take the last image of redis (docker site): I change image: redis:5.0 by image: redis

I purge all images/containers (It's a testing server )

docker ps -a -q | xargs docker rm
docker rmi $(docker images -q)

docker-compose up

and redis work fine

But I have a new message !!! from openvas: ERROR: (ospd_openvas.daemon) OpenVAS Scanner failed to load NVTs. Command '['openvas', '--update-vt-info']' died with <Signals.SIGSEGV: 11>.

Oups, this is the correct option -vt- not -nvt- I think there is a typo in this command: update-vt-info normaly it's update-nvt-info but I don't know how to change this ?

avancerapide commented 4 years ago

After docker-compose up, I connect thru an other tty to create manually a scanner and it's working !! The Web app is OK !!

Thanks for all Have a good day

docker exec -it gvm-containers_gvmd_1 bash

root@cb5e74780e9f:/# gvmd --create-scanner="TEST OPENVAS Scanner" --scanner-type="OpenVas" --scanner-host=/var/run/ospd/ospd.sock
md   main:MESSAGE:2020-03-27 14h52.43 utc:86:    Greenbone Vulnerability Manager version 9.0.0 (DB revision 221)
md manage:   INFO:2020-03-27 14h52.43 utc:86:    Creating scanner.
md manage:WARNING:2020-03-27 14h52.43 utc:86: database must be initialised from scanner
util gpgme:MESSAGE:2020-03-27 14h52.44 utc:86: Setting GnuPG dir to '/var/lib/gvm/gvmd/gnupg'
util gpgme:MESSAGE:2020-03-27 14h52.44 utc:86: Using OpenPGP engine version '2.2.4'
Scanner created.
root@cb5e74780e9f:/# gvmd --get-scanners
md   main:MESSAGE:2020-03-27 14h53.12 utc:113:    Greenbone Vulnerability Manager version 9.0.0 (DB revision 221)
md manage:   INFO:2020-03-27 14h53.12 utc:113:    Getting scanners.
md manage:WARNING:2020-03-27 14h53.12 utc:113: database must be initialised from scanner
08b69003-5fc2-4037-a479-93b440211c73  OpenVAS Default
6acd0832-df90-11e4-b9d5-28d24461215b  CVE
1e826017-850c-409f-8570-6f55054dab7d  TEST OPENVAS Scanner
root@cb5e74780e9f:/# gvmd --verify-scanner=1e826017-850c-409f-8570-6f55054dab7d
md   main:MESSAGE:2020-03-27 14h53.36 utc:120:    Greenbone Vulnerability Manager version 9.0.0 (DB revision 221)
md manage:   INFO:2020-03-27 14h53.36 utc:120:    Verifying scanner.
md manage:WARNING:2020-03-27 14h53.36 utc:120: database must be initialised from scanner
util gpgme:MESSAGE:2020-03-27 14h53.36 utc:120: Setting GnuPG dir to '/var/lib/gvm/gvmd/gnupg'
util gpgme:MESSAGE:2020-03-27 14h53.36 utc:120: Using OpenPGP engine version '2.2.4'
Scanner version: OpenVAS 7.0.0.
root@cb5e74780e9f:/#

admirito commented 4 years ago

The docker-compose has updated in GH-10.

admirito / gvm-containers

update docker-compose #9