Unable to download trivy vulnerability DB

[Lyrwon]

Hi, i start a new issue with an old problem:

I get this message if i want scan a pushed image with trivy:

2021-02-02T07:12:40Z [ERROR] [/pkg/scan/job.go:284]: check scan report with mime type application/vnd.scanner.adapter.vuln.report.harbor+json; version=1.0: running trivy wrapper: running trivy: exit status 1: 2021-02-02T07:12:30.142Z   INFO   Need to update DB
2021-02-02T07:12:30.143Z    INFO   Downloading DB...
2021-02-02T07:12:40.150Z    FATAL  failed to download vulnerability DB: failed to download vulnerability DB: failed to list releases: Get https://api.github.com/repos/aquasecurity/trivy-db/releases: dial tcp: lookup api.github.com on 127.0.0.11:53: read udp 127.0.0.1:38662->127.0.0.11:53: i/o timeout
: general response handler: unexpected status code: 500, expected: 200

I have started harbor on an Ubunut 18.04 VMBox and an Ubuntu 20.04 VMBox, both running on Windows 10. I have tried the same config on a nativ Ubuntu 18.04 but there is the same problem.

I hope anyone of you can help me

version: '2.3'
services:
  log:
    image: goharbor/harbor-log:v2.1.3
    container_name: harbor-log
    restart: always
    dns_search: .
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - DAC_OVERRIDE
      - SETGID
      - SETUID
    volumes:
      - /opt/harbor/var/log/harbor/:/var/log/docker/:z
      - type: bind
        source: /opt/harbor/common/config/log/logrotate.conf
        target: /etc/logrotate.d/logrotate.conf
      - type: bind
        source: /opt/harbor/common/config/log/rsyslog_docker.conf
        target: /etc/rsyslog.d/rsyslog_docker.conf
    ports:
      - 127.0.0.1:1514:10514
    networks:
      - harbor
  registry:
    image: goharbor/registry-photon:v2.1.3
    container_name: registry
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
    volumes:
      - /opt/harbor/data/registry:/storage:z
      - /opt/harbor/common/config/registry/:/etc/registry/:z
      - type: bind
        source: /opt/harbor/data/secret/registry/root.crt
        target: /etc/registry/root.crt
      - type: bind
        source: /opt/harbor/common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      - harbor
      - harbor-clair
    dns_search: .
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "registry"
  registryctl:
    image: goharbor/harbor-registryctl:v2.1.3
    container_name: registryctl
    env_file:
      - /opt/harbor/common/config/registryctl/env
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
    volumes:
      - /opt/harbor/data/registry:/storage:z
      - /opt/harbor/common/config/registry/:/etc/registry/:z
      - type: bind
        source: /opt/harbor/common/config/registryctl/config.yml
        target: /etc/registryctl/config.yml
      - type: bind
        source: /opt/harbor/common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      - harbor
    dns_search: .
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "registryctl"
  postgresql:
    image: goharbor/harbor-db:v2.1.3
    container_name: harbor-db
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - DAC_OVERRIDE
      - SETGID
      - SETUID
    volumes:
      - /opt/harbor/data/database:/var/lib/postgresql/data:z
    networks:
      harbor:
      harbor-clair:
        aliases:
          - harbor-db
    dns_search: .
    env_file:
      - /opt/harbor/common/config/db/env
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "postgresql"
  core:
    image: goharbor/harbor-core:v2.1.3
    container_name: harbor-core
    env_file:
      - /opt/harbor/common/config/core/env
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - SETGID
      - SETUID
    volumes:
      - /opt/harbor/data/ca_download/:/etc/core/ca/:z
      - /opt/harbor/data/:/data/:z
      - /opt/harbor/common/config/core/certificates/:/etc/core/certificates/:z
      - type: bind
        source: /opt/harbor/common/config/core/app.conf
        target: /etc/core/app.conf
      - type: bind
        source: /opt/harbor/data/secret/core/private_key.pem
        target: /etc/core/private_key.pem
      - type: bind
        source: /opt/harbor/data/secret/keys/secretkey
        target: /etc/core/key
      - type: bind
        source: /opt/harbor/common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      harbor:
      harbor-clair:
        aliases:
          - harbor-core
    dns_search: .
    depends_on:
      - log
      - registry
      - redis
      - postgresql
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "core"
  portal:
    image: goharbor/harbor-portal:v2.1.3
    container_name: harbor-portal
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
    volumes:
      - type: bind
        source: /opt/harbor/common/config/portal/nginx.conf
        target: /etc/nginx/nginx.conf
    networks:
      - harbor
    dns_search: .
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "portal"

  jobservice:
    image: goharbor/harbor-jobservice:v2.1.3
    container_name: harbor-jobservice
    env_file:
      - /opt/harbor/common/config/jobservice/env
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
    volumes:
      - /opt/harbor/data/job_logs:/var/log/jobs:z
      - type: bind
        source: /opt/harbor/common/config/jobservice/config.yml
        target: /etc/jobservice/config.yml
      - type: bind
        source: /opt/harbor/common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      - harbor
      - harbor-clair
    dns_search: .
    depends_on:
      - core
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "jobservice"
  redis:
    image: goharbor/redis-photon:v2.1.3
    container_name: redis
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
    volumes:
      - /opt/harbor/data/redis:/var/lib/redis
    networks:
      harbor:
      harbor-clair:
        aliases:
          - redis
    dns_search: .
    depends_on:
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "redis"
  proxy:
    image: goharbor/nginx-photon:v2.1.3
    container_name: nginx
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID
      - NET_BIND_SERVICE
    volumes:
      - /opt/harbor/common/config/nginx:/etc/nginx:z
      - type: bind
        source: /opt//harbor/common/config/shared/trust-certificates
        target: /harbor_cust_cert
    networks:
      - harbor
    dns_search: .
    ports:
      - 80:8080
    depends_on:
      - registry
      - core
      - portal
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "proxy"
  clair:
    networks:
      - harbor-clair
    container_name: clair
    image: goharbor/clair-photon:v2.1.3
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - DAC_OVERRIDE
      - SETGID
      - SETUID
    cpu_quota: 50000
    dns_search: .
    depends_on:
      - log
      - postgresql
    volumes:
      - type: bind
        source: /opt/harbor/common/config/clair/config.yaml
        target: /etc/clair/config.yaml
      - type: bind
        source: /opt/harbor/common/config/shared/trust-certificates
        target: /harbor_cust_cert
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "clair"
    env_file:
      /opt/harbor/common/config/clair/clair_env
  clair-adapter:
    networks:
      - harbor-clair
    container_name: clair-adapter
    image: goharbor/clair-adapter-photon:v2.1.3
    restart: always
    cap_drop:
      - ALL
    cap_add:
      - DAC_OVERRIDE
      - SETGID
      - SETUID
    cpu_quota: 50000
    dns_search: .
    depends_on:
      - clair
      - redis
    volumes:
      - type: bind
        source: /opt/harbor/common/config/shared/trust-certificates
        target: /harbor_cust_cert
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "clair-adapter"
    env_file:
      /opt/harbor/common/config/clair-adapter/env
  trivy-adapter:
    container_name: trivy-adapter
    image: goharbor/trivy-adapter-photon:v2.1.3
    restart: always
    cap_drop:
      - ALL
    dns_search: .
    depends_on:
      - log
      - redis
    networks:
      - harbor
    volumes:
      - type: bind
        source: /opt/harbor/data/trivy-adapter/trivy
        target: /home/scanner/.cache/trivy
      - type: bind
        source: /opt/harbor/data/trivy-adapter/reports
        target: /home/scanner/.cache/reports
      - type: bind
        source: /opt/harbor/common/config/shared/trust-certificates
        target: /harbor_cust_cert
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "trivy-adapter"
    env_file:
      /opt/harbor/common/config/trivy-adapter/env
networks:
  harbor:
    external: false
  harbor-clair:
    external: false

[heww]

According to the log message dial tcp: lookup api.github.com on 127.0.0.11:53: read udp 127.0.0.1:38662->127.0.0.11:53: i/o timeout, please check your env of docker daemon, there are some issues for the DNS lookup of the docker daemon.

There is a DNS issue https://github.com/moby/moby/issues/31960 about the docker daemon.

[sluetze]

it might be worth a shot to try a docker version <20 which do not have that issue

[Lyrwon]

According to the log message dial tcp: lookup api.github.com on 127.0.0.11:53: read udp 127.0.0.1:38662->127.0.0.11:53: i/o timeout, please check your env of docker daemon, there are some issues for the DNS lookup of the docker daemon.

There is a DNS issue moby/moby#31960 about the docker daemon.

I changed the /etc/resolv.conf nameserver 8.8.8.8 options edns0

and add a volume:

trivy-adapter:
...
volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
      ...

[bitsf]

docker exec -it trivy-adapter bash and ping api.github.com, make sure it can be successful try restart docker service or machine

[dustyhorizon]

seems like the generated docker-compose.yml hardcodes the services to use the harbor network and it is defined as an "internal" only network, my current workaround is to add the default network in addition to the harbor network as well as removing the dns_search: . entry for the relevant harbor service that requires internet access

[Lyrwon]

Thank you for your help and ideas.

My current workaround is to add the /etc/resolv.conf as a volume in the data directory and if their is a conneciton error I start this bashscript.

#!/bin/bash
printf "nameserver 8.8.8.8\noptions ndots:0" > /opt/harbor/data/trivy-adapter/resolv.conf;
docker-compose up -d --force-recreate trivy-adapter;
docker exec -it trivy-adapter trivy fs /;
printf "nameserver 127.0.0.11\noptions ndots:0" > /opt/harbor/data/trivy-adapter/resolv.conf;
docker-compose up -d --force-recreate trivy-adapter;
docker exec -it trivy-adapter trivy fs /;

I don't know why, but it wokrs ¯_(ツ)_/¯

[morsik]

@bitsf why you closed this issue? Is it really resolved now?

I installed Harbor few days ago and went into same issue. And removing dns_search fixed things for me. Why is it even used then?

[Lyrwon]

Unfortunately it doesn't work again and it run into the same error.

After i edit the docker-compose.yml, it works for the moment

. . .
cap_drop:
      - ALL
#    dns_search: .
    depends_on:
      - log
      - redis
    networks:
. . .

[wy65701436]

It's introduced by docker dns change(https://github.com/moby/moby/pull/41022), please see https://github.com/goharbor/harbor/issues/13718#issuecomment-744179993 to get the workaround. And refer to https://github.com/moby/moby/issues/41819 to get some backgrounds.

To remove dns_search: . is not an acceptable solution, as the setting is to disable host dns server to be mounted into container.

Think about the scenario, harbor core is searching postgres(which is harbor db) on launching, if the host dns is mounted into harbor core and there is a postgres which is reachable in its network. The harbor core will try to connect to the first reachable postgres host, but in some cases, this postgres may not the harbor db container.

[wy65701436]

hi @morsik we can use #13718 for tracking the problem, and use the workaround mentioned above to resolve. Just keep this issue close.

[Lyrwon]

@wy65701436 maby it is not acceptable, but it works!

Do you have a reason why it won't work without removeing?

Or better, do you have a reason to solv this problem?

[mk3rnb8]

@Lyrwon after I commented all dns_search entries out in the docker-compose.yml I get a scanner [ / ]$ ping api.github.com ping: socket SOCK_DGRAM 1: Permission denied Kernel bug workaround: echo 0 99999999 | sudo tee /proc/sys/net/ipv4/ping_group_range

So, is there a stable solution for Harbor? As I do not see any, why this has been closed?

[Colleoni]

@mk3rnb8 as far as I understood you get the error because you have no permission to use ping, it is a non-root docker container. However, the solution will probably work, because with standard configuration you will instead receive a timeout.

@wy65701436 sorry I get and idea but I do not fully understand your point in not removing dns_serach: . I followed the instructions in #13718 (comment) and they correctly work. However, if I correctly understood using it we are providing the dns configuration we set in daemon.json to all containers, thus it seems a work around but with the same effect, the container will use basically the host dns. The only difference I see is that you can set a dns different than the host dns, is that correct?

[wy65701436]

@colleoni To remove the dns_serach: . will introduce the issue mentioned here

[thirisangu84]

docker exec -it trivy-adapter trivy fs /;

I tried the same but not working, I am trying with 2.5.0

harbor]# docker exec -it trivy-adapter trivy fs /; 2022-10-17T11:58:34.416Z INFO Need to update DB 2022-10-17T11:58:34.416Z INFO Downloading DB... 2022-10-17T11:58:35.030Z FATAL DB error: failed to download vulnerability DB: OCI artifact error: OCI artifact error: OCI repository error: Get "https://ghcr.io/v2/": read tcp 172.23.0.8:52042->xx.xxx.xxx.x:8080: read: connection reset by peer harbor]#