gitlab-runner, stuck jobs in creating mode (500 internal server error)

ma0dubois commented 5 years ago

Hello everyone,

I have a problem that I can not solve alone. I spent the day on it and I tried to find a solution by myself before creating this ticket.

Context:

I use the latest version of Gitlab proposed: sameersbn / gitlab: 11.5.1
I have a runner launched into a docker container: gitlab / gitlab-runner: alpine
I use Traefik

Everything is started thanks to docker-compose.

Steps:

I launch all of my containers
I record a runner (command visible below)
I notice in the admin that my runners are well recorded in the gitlab: / admin / runners

recorded

I run a pipeline, and the job is blocked: /

Job is blocked

I have tried everything: - update the gitlab - update the runner and use a previous version - remove the runner from the gitlab network - ...

Details:

My Traefik docker-compose.yml:

version: '2'

services:
  proxy:
    image: traefik:alpine
    container_name: traefik
    networks:
      - traefik
    ports:
      - "80:80"
      - "8080:8080"
      - "443:443"
    volumes:
      - /data/traefik/traefik.toml:/etc/traefik/traefik.toml
      - /data/traefik/acme:/etc/traefik/acme
      - /var/run/docker.sock:/var/run/docker.sock
    restart: unless-stopped
    labels:
      - "traefik.frontend.rule=Host:traefik.mydomain.com"
      - "traefik.port=8080"
      - "traefik.backend=traefik"
      - "traefik.frontend.entryPoints=http,https"

  portainer:
    image: portainer/portainer
    container_name: portainer
    networks:
      - traefik
    labels: 
      - "traefik.frontend.rule=Host:portainer.mydomain.com"
      - "traefik.port=9000"
      - "traefik.backend=portainer"
      - "traefik.frontend.entryPoints=http,https"
    volumes:
        - /var/run/docker.sock:/var/run/docker.sock
    restart: unless-stopped

networks:
  traefik:
    external:
      name: traefik

My Gitlab docker-compose.yml:

version: '2'

services:
  redis:
    restart: always
    image: sameersbn/redis:4.0.9-1
    container_name: gitlab-redis
    command:
      - --loglevel warning
    networks:
      - gitlab
    volumes:
      - /data/gitlab/redis:/var/lib/redis:Z
    labels:
      - "traefik.enable=false"

  postgresql:
    restart: always
    image: sameersbn/postgresql:10
    container_name: gitlab-postgresql
    networks:
      - gitlab
    volumes:
      - /data/gitlab/postgresql:/var/lib/postgresql:Z
    environment:
      - DB_USER=gitlab
      - DB_PASS=password
      - DB_NAME=gitlabhq_production
      - DB_EXTENSION=pg_trgm
    labels:
      - "traefik.enable=false"

  registry:
    image: registry:2
    container_name: gitlab-registry
    restart: always
    expose:
        - "5000"
    ports:
        - "5000:5000"
    networks:
      - gitlab
      - traefik
    volumes:
        - /data/gitlab/registry:/registry
        - /data/gitlab/certs:/certs
    labels:
      - traefik.port=5000
      - traefik.frontend.rule=Host:registry.mydomain.com
      - traefik.frontend.auth.basic=mydomain:fd9ef338f7de0f196c5409a668102b9a
    environment:
        - REGISTRY_LOG_LEVEL=error
        - REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/registry
        - REGISTRY_AUTH_TOKEN_REALM=https://gitlab.mydomain.com/jwt/auth
        - REGISTRY_AUTH_TOKEN_SERVICE=container_registry
        - REGISTRY_AUTH_TOKEN_ISSUER=gitlab-issuer
        - REGISTRY_AUTH_TOKEN_ROOTCERTBUNDLE=/certs/registry.crt
        - REGISTRY_STORAGE_DELETE_ENABLED=true

  gitlab-runner:
    image: gitlab/gitlab-runner:alpine
    container_name: gitlab-runner
    restart: always
    depends_on:
      - gitlab
    networks:
      - gitlab
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /data/gitlab-runner:/etc/gitlab-runner:Z
    environment:
      - CI_SERVER_URL=https://gitlab.mydomain.com/
      - REGISTRATION_TOKEN=FzZtgyN1cAMzoYne89ts
    labels:
      - "traefik.enable=false"

  gitlab:
    restart: always
    image: sameersbn/gitlab:11.5.1
    container_name: gitlab-ce
    depends_on:
      - redis
      - postgresql
      - registry
    ports:
      - "10080:80"
      - "10022:22"
    networks:
      - gitlab
      - traefik
    volumes:
      - /data/gitlab/gitlab:/home/git/data:Z
      - /data/gitlab/certs:/certs
    environment:
      - DEBUG=false

      - DB_ADAPTER=postgresql
      - DB_HOST=postgresql
      - DB_PORT=5432
      - DB_USER=gitlab
      - DB_PASS=password
      - DB_NAME=gitlabhq_production

      - REDIS_HOST=redis
      - REDIS_PORT=6379

      - TZ=Europe/Paris
      - GITLAB_TIMEZONE=Paris

      - GITLAB_HTTPS=true
      - SSL_SELF_SIGNED=false

      - GITLAB_HOST=gitlab.mydomain.com
      - GITLAB_PORT=
      - GITLAB_SSH_PORT=10022
      - GITLAB_RELATIVE_URL_ROOT=
      - GITLAB_SECRETS_DB_KEY_BASE=w58HODDUerP7YOuAbt9heD9j6s80P5A8POUdsd4wHeh7tLU8wdSG0noq2LsRnvqsff9btHJDovejeTMWflg78tvKqT7y9omqVTvh
      - GITLAB_SECRETS_SECRET_KEY_BASE=w58HODDUerP7YOuAbt9heD9j6s80P5A8POUdsd4wHeh7tLU8wdSG0noq2LsRnvqsff9btHJDovejeTMWflg78tvKqT7y9omqVTvh
      - GITLAB_SECRETS_OTP_KEY_BASE=w58HODDUerP7YOuAbt9heD9j6s80P5A8POUdsd4wHeh7tLU8wdSG0noq2LsRnvqsff9btHJDovejeTMWflg78tvKqT7y9omqVTvh

      - GITLAB_ROOT_PASSWORD=
      - GITLAB_ROOT_EMAIL=

      - GITLAB_NOTIFY_ON_BROKEN_BUILDS=true
      - GITLAB_NOTIFY_PUSHER=false

      - GITLAB_EMAIL=notifications@example.com
      - GITLAB_EMAIL_REPLY_TO=noreply@example.com
      - GITLAB_INCOMING_EMAIL_ADDRESS=reply@example.com

      - GITLAB_BACKUP_SCHEDULE=daily
      - GITLAB_BACKUP_TIME=01:00

      - SMTP_ENABLED=false
      - SMTP_DOMAIN=www.example.com
      - SMTP_HOST=smtp.gmail.com
      - SMTP_PORT=587
      - SMTP_USER=mailer@example.com
      - SMTP_PASS=password
      - SMTP_STARTTLS=true
      - SMTP_AUTHENTICATION=login

      - IMAP_ENABLED=false
      - IMAP_HOST=imap.gmail.com
      - IMAP_PORT=993
      - IMAP_USER=mailer@example.com
      - IMAP_PASS=password
      - IMAP_SSL=true
      - IMAP_STARTTLS=false

      - OAUTH_ENABLED=false
      - OAUTH_AUTO_SIGN_IN_WITH_PROVIDER=
      - OAUTH_ALLOW_SSO=
      - OAUTH_BLOCK_AUTO_CREATED_USERS=true
      - OAUTH_AUTO_LINK_LDAP_USER=false
      - OAUTH_AUTO_LINK_SAML_USER=false
      - OAUTH_EXTERNAL_PROVIDERS=

      - OAUTH_CAS3_LABEL=cas3
      - OAUTH_CAS3_SERVER=
      - OAUTH_CAS3_DISABLE_SSL_VERIFICATION=false
      - OAUTH_CAS3_LOGIN_URL=/cas/login
      - OAUTH_CAS3_VALIDATE_URL=/cas/p3/serviceValidate
      - OAUTH_CAS3_LOGOUT_URL=/cas/logout

      - OAUTH_GOOGLE_API_KEY=
      - OAUTH_GOOGLE_APP_SECRET=
      - OAUTH_GOOGLE_RESTRICT_DOMAIN=

      - OAUTH_FACEBOOK_API_KEY=
      - OAUTH_FACEBOOK_APP_SECRET=

      - OAUTH_TWITTER_API_KEY=
      - OAUTH_TWITTER_APP_SECRET=

      - OAUTH_GITHUB_API_KEY=
      - OAUTH_GITHUB_APP_SECRET=
      - OAUTH_GITHUB_URL=
      - OAUTH_GITHUB_VERIFY_SSL=

      - OAUTH_GITLAB_API_KEY=
      - OAUTH_GITLAB_APP_SECRET=

      - OAUTH_BITBUCKET_API_KEY=
      - OAUTH_BITBUCKET_APP_SECRET=

      - OAUTH_SAML_ASSERTION_CONSUMER_SERVICE_URL=
      - OAUTH_SAML_IDP_CERT_FINGERPRINT=
      - OAUTH_SAML_IDP_SSO_TARGET_URL=
      - OAUTH_SAML_ISSUER=
      - OAUTH_SAML_LABEL="Our SAML Provider"
      - OAUTH_SAML_NAME_IDENTIFIER_FORMAT=urn:oasis:names:tc:SAML:2.0:nameid-format:transient
      - OAUTH_SAML_GROUPS_ATTRIBUTE=
      - OAUTH_SAML_EXTERNAL_GROUPS=
      - OAUTH_SAML_ATTRIBUTE_STATEMENTS_EMAIL=
      - OAUTH_SAML_ATTRIBUTE_STATEMENTS_NAME=
      - OAUTH_SAML_ATTRIBUTE_STATEMENTS_FIRST_NAME=
      - OAUTH_SAML_ATTRIBUTE_STATEMENTS_LAST_NAME=

      - OAUTH_CROWD_SERVER_URL=
      - OAUTH_CROWD_APP_NAME=
      - OAUTH_CROWD_APP_PASSWORD=

      - OAUTH_AUTH0_CLIENT_ID=
      - OAUTH_AUTH0_CLIENT_SECRET=
      - OAUTH_AUTH0_DOMAIN=

      - OAUTH_AZURE_API_KEY=
      - OAUTH_AZURE_API_SECRET=
      - OAUTH_AZURE_TENANT_ID=

      - GITLAB_REGISTRY_ENABLED=true
      - GITLAB_REGISTRY_HOST=registry.mydomain.com
      - GITLAB_REGISTRY_API_URL=http://localhost:5000
      - GITLAB_REGISTRY_KEY_PATH=/certs/registry.key
      - GITLAB_REGISTRY_ISSUER=gitlab-issuer
    labels:
      - "traefik.frontend.rule=Host:gitlab.mydomain.com"
      - "traefik.port=80"
      - "traefik.backend=gitlab"
      - "traefik.frontend.entryPoints=http,https"
      - "traefik.docker.network=traefik"

networks:
  gitlab:
    driver: bridge
  traefik:
    external:
      name: traefik

Command for register my runner:

docker exec -it gitlab-runner gitlab-runner register \
  --non-interactive \
  --name "Doker runner dind 1" \
  --url "https://gitlab.mydomain.com/" \
  --registration-token "FzZtgyN1cAMzoYne89ts" \
  --env "COMPOSER_CACHE_DIR=/cache" \
  --env "GIT_SSL_NO_VERIFY=true" \
  --env "DOCKER_DRIVER=overlay2" \
  --executor "docker" \
  --docker-image docker:stable-dind \
  --docker-privileged="true" \
  --docker-volumes /var/run/docker.sock:/var/run/docker.sock \
  --docker-volumes /cache \
  --tag-list "docker,dind" \
  --run-untagged \
  --locked="false"

gjrtimmer commented 5 years ago

Had this a while ago. Any change your gitlab-runner is not up to date and still trying to connect to the old CI URL.

Somewhere in the 11 series it was changed. Check out your admin section and verify that the runners are connecting to the right URL.

Try my gitlab-runner image https://github.com/GJRTimmer/docker-gitlab-runner you can find it on docker hub or my personal gitlab. Don't look at the tags.

Currently this image on on a schedule to be auto generated every month on the 25th. On the 22nd there is a gitlab-runner release. This image is always up to date with the latest version.

Hope this helps.

CrystalMethod commented 5 years ago

@ma0dubois can you login to your registry using the docker cli? Since 1-2 weeks I have a very similar error. The combination of Gitlab, Registry and Traefik went well over at least 1 year and currently I don't know what the cause is.

$ echo ${CI_REGISTRY_PASS} | docker login -u ${CI_REGISTRY_USER} --password-stdin ${CI_REGISTRY}
Error response from daemon: Get https://registry.***.com/v2/: received unexpected HTTP status: 500 Internal Server Error

docker login registry.***.com                                                                                                                                                                                                   
Authenticating with existing credentials...
Login did not succeed, error: Error response from daemon: Get https://registry.***.com/v2/: received unexpected HTTP status: 500 Internal Server Error
Username (***): ***
Password:
Error response from daemon: Get https://registry.***.com/v2/: received unexpected HTTP status: 500 Internal Server Error

ma0dubois commented 5 years ago

@GJRTimmer @CrystalMethod

Sorry for my late response.

I have long investigated, without success: / So I started to put me a gitlab with the native Docker image of gitlab, and it works today!

If it interests someone I can provide my docker-composer.yml with:

traefik
portainer
a listener of the change of the certificates let's encrypt which automatically exports the .crt and .key to bind them to the gitlab and the registry
a docker registry
gitlab-ce
a runner

Thanks anyway for your answers :)

elielam commented 5 years ago

@ma0dubois can you provide us your docker-compose please. Stuck with same issue since 3months now :)

ma0dubois commented 5 years ago

Hi,

Yes, with pleasure ☺️

version: '3'

networks:
  proxy:
    external: true
  internal:
    external: false

services:

  # user: myuser
  # mdp:  9EwzxWGzqvY4aSH3
  traefik:
    image: traefik:alpine
    container_name: traefik
    networks:
      - proxy
    ports:
      - "80:80"
      - "8080:8080"
      - "443:443"
    volumes:
      - /data/traefik/traefik.toml:/etc/traefik/traefik.toml
      - /data/traefik/acme:/etc/traefik/acme
      - /var/run/docker.sock:/var/run/docker.sock
    restart: unless-stopped
    labels:
      - traefik.frontend.rule=Host:traefik.company.com
      - traefik.port=8080
      - traefik.backend=traefik
      - traefik.frontend.entryPoints=http,https

  # Watch acme.json and dump certificates to files
  certdumper:
    container_name: traefik_certdumper
    image: alpine:latest
    depends_on:
      - traefik
    restart: unless-stopped
    volumes:
      - /data/traefik:/traefik
    command: >
      ash -c " \
        apk --no-cache add inotify-tools jq openssl util-linux bash && \
        wget https://raw.githubusercontent.com/containous/traefik/master/contrib/scripts/dumpcerts.sh -O dumpcerts.sh && \
        mkdir -p /traefik/ssl/ && \
        while true; do \
          inotifywait -e modify /traefik/acme/acme.json && \
            bash dumpcerts.sh /traefik/acme/acme.json /traefik/ssl/ && \
            rm -f /traefik/ssl/*.crt && ln -f /traefik/ssl/certs/* /traefik/ssl/ && \
            rm -f /traefik/ssl/*.key && ln -f /traefik/ssl/private/* /traefik/ssl/ && \
            chmod 644 -R /traefik/ssl;
        done"

  # user: devteam
  # mdp:  9EwzxWGzqvY4aSH3
  portainer:
    image: portainer/portainer:latest
    container_name: portainer
    networks:
      - proxy
    labels:
      - traefik.enable=true
      - traefik.frontend.rule=Host:portainer.company.com
      - traefik.port=9000
      - traefik.backend=portainer
      - traefik.frontend.entryPoints=http,https
    volumes:
        - /var/run/docker.sock:/var/run/docker.sock
        - portainer-data:/data
    restart: unless-stopped

  registry:
    image: registry:2
    container_name: gitlab-registry
    restart: always
    networks:
      - proxy
      - internal
    depends_on:
      - traefik
    expose:
        - "5000"
    ports:
        - "5000:5000"
    volumes:
        - /data/gitlab/registry:/registry
        - /data/traefik/ssl:/certs
    labels:
      - traefik.enable=true
      - traefik.port=5000
      - traefik.frontend.rule=Host:registry.company.com
      - traefik.docker.network=proxy
      # - traefik.frontend.auth.basic=myuser:fd9ef338f7de0f196c5409a668102b9a
    environment:
      - REGISTRY_LOG_LEVEL=error
      - REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY=/registry
      - REGISTRY_AUTH_TOKEN_REALM=https://gitlab.company.com/jwt/auth
      - REGISTRY_AUTH_TOKEN_SERVICE=container_registry
      - REGISTRY_AUTH_TOKEN_ISSUER=gitlab-issuer
      - REGISTRY_AUTH_TOKEN_ROOTCERTBUNDLE=/certs/registry.company.com.crt
      - REGISTRY_STORAGE_DELETE_ENABLED=true

  gitlab-runner:
    image: gitlab/gitlab-runner:alpine
    container_name: gitlab-runner
    networks:
      - proxy
      - internal
    restart: always
    depends_on:
      - gitlab
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - /data/gitlab-runner:/etc/gitlab-runner:Z
    environment:
      - CI_SERVER_URL=https://gitlab.company.com/
      - REGISTRATION_TOKEN=FzZtgyN1cAMzoYne89ts

  gitlab:
    restart: always
    image: gitlab/gitlab-ce:11.5.3-ce.0
    container_name: gitlab-ce
    hostname: gitlab.company.com
    networks:
      - proxy
      - internal
    healthcheck:
      disable: true
    depends_on:
      - traefik
      - registry
    ports:
      - "2222:22"
    depends_on:
      - registry
    volumes:
      - /data/gitlab/gitlab-config:/etc/gitlab
      - /data/gitlab/gitlab-logs:/var/log/gitlab
      - /data/gitlab/gitlab-data:/var/opt/gitlab
      - /data/traefik/ssl:/certs
    environment:
      GITLAB_OMNIBUS_CONFIG: |
        # External URL as it will be seen by GitLab users, so with HTTPS, even if GitLab itself only serves HTTP
        external_url 'https://gitlab.company.com'

        # Disable HTTPS and serve plain HTTP. The rest will be handled by Traefik
        nginx['listen_port'] = 80
        nginx['listen_https'] = false

        # I had some issues and therefore disabled HTTP/2, but should normally work
        nginx['http2_enabled'] = false

        # Pass headers to GitLab, $$ escapes 
        nginx['proxy_set_headers'] = {
          "Host" => "$$http_host",
          "X-Real-IP" => "$$remote_addr",
          "X-Forwarded-For" => "$$proxy_add_x_forwarded_for",
          "X-Forwarded-Proto" => "https",
          "X-Forwarded-Ssl" => "on"
        }

        # Define SSH port for git+ssh, can also be changed as you like
        gitlab_rails['gitlab_shell_ssh_port'] = 2222

        # External URL for the registry as seen by GitLab users with HTTPS even if Gitlab itself only serves HTTP
        registry_external_url 'https://registry.company.com'

        gitlab_rails['registry_enabled'] = true
        gitlab_rails['registry_key_path'] = '/certs/registry.company.com.key'
        gitlab_rails['registry_api_url'] = 'http://gitlab-registry:5000'
        gitlab_rails['registry_issuer'] = 'gitlab-issuer'

        registry_nginx['ssl_certificate'] = "/certs/registry.company.com.crt"
        registry_nginx['ssl_certificate_key'] = "/certs/registry.company.com.key"

        # External URL for Pages hosting as seen by GitLab users with HTTPS even if Gitlab itself only serves HTTP
        pages_external_url 'https://pages.gitlab.company.com'

        # Disable HTTPS and set custom port for the service
        pages_nginx['listen_port'] = 5200
        pages_nginx['listen_https'] = false

        # Pass headers
        pages_nginx['proxy_set_headers'] = {
          "Host" => "$$http_host",
          "X-Real-IP" => "$$remote_addr",
          "X-Forwarded-For" => "$$proxy_add_x_forwarded_for",
          "X-Forwarded-Proto" => "https",
          "X-Forwarded-Ssl" => "on"
        }

        # Seems like when you use Docker data volumes, you need this, otherwise it shows this in the log:
        # "Failed to bind mount /gitlab-data/shared/pages on /tmp/gitlab-pages-xyz/pages. operation not permitted"
        gitlab_pages['inplace_chroot'] = true

        # Tell GitLab to use an external HTTP server, like Traefik in our case, to handle custom domains.
        # The documentation says that you'd need to point an additional IP address here that you want to use but I 
        # figured out that just using the GitLab container name works as well, so no need to purchase an additional
        # domain :-)
        gitlab_pages['external_http'] = ['gitlab:5201']
    labels:
      - traefik.enable=true
      - traefik.frontend.rule=Host:gitlab.company.com
      - traefik.docker.network=proxy
      - traefik.port=80
      - traefik.backend=gitlab
      - traefik.frontend.entryPoints=http,https

volumes:
  portainer-data:
    driver: local

sameersbn / docker-gitlab

gitlab-runner, stuck jobs in creating mode (500 internal server error) #1803