search

pulp / pulp-oci-images

Containerfiles and other assets for building Pulp 3 OCI images
https://docs.pulpproject.org/pulp_oci_images/
29 stars 53 forks source link

DB Migration fails with ERROR: relation "core_artifact" does not exist at character 28 #641

Open jeremyatourville opened 1 month ago

jeremyatourville commented 1 month ago

Version docker.io/library/postgres:13 docker.io/library/redis:latest pulp/pulp-minimal:latest pulp/pulp-web:latest

Describe the bug service "migration_service" didn't completed successfully: exit 139 is seen when running docker-compose up -d docker logs show this line - ERROR: relation "core_artifact" does not exist at character 28

To Reproduce Use compose.folders.yaml from git mkdir ./pgsql ./pulp_storage sudo chown 700:700 ./pulp_storage sudo chown 26:26 ./pgsql

Expected behavior DB Migration should complete without issues

Additional context I am running Rocky 8.9 and Docker/Docker-Compose My system is air-gapped and I am pulling images from my private repo.

Folder structure:

pulp/
|- assets/
|- pgsql/
|- pulp_storage/
|-  docker-compose.yml (compose-folder.yaml)
[root@gsil-docker1 pulp]# cat docker-compose.yml 
version: '3'
services:
  postgres:
    image: "gsil-docker1.idm.gsil.org:5000/postgres:13"
    ports:
      - "5432:5432"
    environment:
      POSTGRES_USER: pulp
      POSTGRES_PASSWORD: password
      POSTGRES_DB: pulp
      POSTGRES_INITDB_ARGS: '--auth-host=scram-sha-256'
      POSTGRES_HOST_AUTH_METHOD: 'scram-sha-256'
    volumes:
      - "./pgsql/data:/var/lib/postgresql/data"
      - "./assets/postgres/passwd:/etc/passwd:Z"
    restart: always
    healthcheck:
      test: pg_isready -U pulp
      interval: 10s
      timeout: 5s
      retries: 5

  redis:
    image: "gsil-docker1.idm.gsil.org:5000/redis:latest"
    volumes:
      - "redis_data:/data"
    restart: always
    healthcheck:
      test: redis-cli ping
      interval: 10s
      timeout: 5s
      retries: 5

  migration_service:
    image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
    depends_on:
      postgres:
        condition: service_healthy
    command: pulpcore-manager migrate --noinput
    volumes:
      - "./assets/settings.py:/etc/pulp/settings.py:z"
      - "./assets/certs:/etc/pulp/certs:z"
      - "./pulp_storage:/var/lib/pulp"

  signing_key_service:
    image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
    command: sh -c "add_signing_service.sh"
    depends_on:
      postgres:
        condition: service_healthy
      migration_service: 
        condition: service_completed_successfully
    volumes:
      - "./assets/settings.py:/etc/pulp/settings.py:z"
      - "./assets/certs:/etc/pulp/certs:z"
      - "./pulp_storage:/var/lib/pulp"

  set_init_password_service:
    image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
    command: set_init_password.sh
    depends_on:
      postgres:
        condition: service_healthy
    environment:
      PULP_DEFAULT_ADMIN_PASSWORD: password
    volumes:
      - "./assets/settings.py:/etc/pulp/settings.py:z"
      - "./assets/certs:/etc/pulp/certs:z"
      - "./pulp_storage:/var/lib/pulp"

  pulp_web:
    image: "gsil-docker1.idm.gsil.org:5000/pulp-web:latest"
    command: ['/usr/bin/nginx.sh']
    depends_on:
      pulp_api:
        condition: service_healthy
      pulp_content:
        condition: service_healthy
    ports:
      - "8081:8080"
    hostname: pulp
    user: root
    volumes:
      - "./assets/bin/nginx.sh:/usr/bin/nginx.sh:Z"
      - "./assets/nginx/nginx.conf.template:/etc/opt/rh/rh-nginx116/nginx/nginx.conf.template:Z"
    restart: always

  pulp_api:
    image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
    deploy:
      replicas: 2
    command: ['pulp-api']
    depends_on:
      redis:
        condition: service_healthy
      postgres:
        condition: service_healthy
      migration_service:
        condition: service_completed_successfully
      set_init_password_service:
        condition: service_completed_successfully
      signing_key_service:
        condition: service_completed_successfully
    hostname: pulp-api
    user: pulp
    volumes:
      - "./assets/settings.py:/etc/pulp/settings.py:z"
      - "./assets/certs:/etc/pulp/certs:z"
      - "./pulp_storage:/var/lib/pulp"
    restart: always
    healthcheck:
      test: readyz.py /pulp/api/v3/status/
      interval: 10s
      timeout: 5s
      retries: 5

  pulp_content:
    image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
    deploy:
      replicas: 2
    command: ['pulp-content']
    depends_on:
      redis:
        condition: service_healthy
      postgres:
        condition: service_healthy
      migration_service:
        condition: service_completed_successfully
    hostname: pulp-content
    user: pulp
    volumes:
      - "./assets/settings.py:/etc/pulp/settings.py:z"
      - "./assets/certs:/etc/pulp/certs:z"
      - "./pulp_storage:/var/lib/pulp"
    restart: always
    healthcheck:
      test: readyz.py /pulp/content/
      interval: 10s
      timeout: 5s
      retries: 5

  pulp_worker:
    image: "gsil-docker1.idm.gsil.org:5000/pulp-minimal:latest"
    deploy:
        replicas: 2
    command: ['pulp-worker']
    depends_on:
      redis:
        condition: service_healthy
      postgres:
        condition: service_healthy
      migration_service:
        condition: service_completed_successfully
    user: pulp
    volumes:
      - "./assets/settings.py:/etc/pulp/settings.py:z"
      - "./assets/certs:/etc/pulp/certs:z"
      - "./pulp_storage:/var/lib/pulp"
    restart: always

volumes:
  redis_data:
    name: redis_data${DEV_VOLUME_SUFFIX:-dev}

[root@gsil-docker1 pulp]# ls -lah
total 12K
drwx------.  5 root     root       79 Jun 11 07:34 .
drwxr-xr-x. 22 root     root     4.0K Jun  7 14:15 ..
drwxr-xr-x.  6 localadm localadm   78 Jun 10 08:02 assets
-rw-r--r--.  1 localadm localadm 4.5K Jun 11 07:34 docker-compose.yml
drwxr-xr-x.  3       26       26   18 Jun 11 07:21 pgsql
drwxr-xr-x.  3      700      700   20 Jun 11 07:22 pulp_storage

[root@gsil-docker1 pulp]# docker-compose up -d
[+] Running 68/68
 ...
[+] Running 13/13
 ⠿ Network pulp_default                        Created                                                                                                                                                        0.3s
 ⠿ Container pulp-postgres-1                   Healthy                                                                                                                                                       13.8s
 ⠿ Container pulp-redis-1                      Healthy                                                                                                                                                       13.8s
 ⠿ Container pulp-migration_service-1          Exited                                                                                                                                                        16.7s
 ⠿ Container pulp-set_init_password_service-1  Started                                                                                                                                                       12.7s
 ⠿ Container pulp-pulp_content-2               Created                                                                                                                                                        0.1s
 ⠿ Container pulp-pulp_worker-2                Created                                                                                                                                                        0.1s
 ⠿ Container pulp-signing_key_service-1        Created                                                                                                                                                        0.1s
 ⠿ Container pulp-pulp_content-1               Created                                                                                                                                                        0.1s
 ⠿ Container pulp-pulp_worker-1                Created                                                                                                                                                        0.1s
 ⠿ Container pulp-pulp_api-2                   Created                                                                                                                                                        0.1s
 ⠿ Container pulp-pulp_api-1                   Created                                                                                                                                                        0.1s
 ⠿ Container pulp-pulp_web-1                   Created                                                                                                                                                        0.0s
service "migration_service" didn't completed successfully: exit 139

pulp-postgres-1                   | The files belonging to this database system will be owned by user "postgres".
pulp-postgres-1                   | This user must also own the server process.
pulp-postgres-1                   | 
pulp-postgres-1                   | The database cluster will be initialized with locale "en_US.utf8".
pulp-redis-1                      | 1:C 11 Jun 2024 12:41:44.727 * oO0OoO0OoO0Oo Redis is starting oO0OoO0OoO0Oo
pulp-redis-1                      | 1:C 11 Jun 2024 12:41:44.727 * Redis version=7.2.5, bits=64, commit=00000000, modified=0, pid=1, just started
pulp-redis-1                      | 1:C 11 Jun 2024 12:41:44.727 # Warning: no config file specified, using the default config. In order to specify a config file use redis-server /path/to/redis.conf
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.728 * monotonic clock: POSIX clock_gettime
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.730 * Running mode=standalone, port=6379.
pulp-postgres-1                   | The default database encoding has accordingly been set to "UTF8".
pulp-postgres-1                   | The default text search configuration will be set to "english".
pulp-postgres-1                   | 
pulp-postgres-1                   | Data page checksums are disabled.
pulp-postgres-1                   | 
pulp-postgres-1                   | fixing permissions on existing directory /var/lib/postgresql/data ... ok
pulp-postgres-1                   | creating subdirectories ... ok
pulp-postgres-1                   | selecting dynamic shared memory implementation ... posix
pulp-postgres-1                   | selecting default max_connections ... 100
pulp-postgres-1                   | selecting default shared_buffers ... 128MB
pulp-postgres-1                   | selecting default time zone ... Etc/UTC
pulp-postgres-1                   | creating configuration files ... ok
pulp-postgres-1                   | running bootstrap script ... ok
pulp-postgres-1                   | performing post-bootstrap initialization ... ok
pulp-postgres-1                   | initdb: warning: enabling "trust" authentication for local connections
pulp-postgres-1                   | You can change this by editing pg_hba.conf or using the option -A, or
pulp-postgres-1                   | --auth-local and --auth-host, the next time you run initdb.
pulp-postgres-1                   | syncing data to disk ... ok
pulp-postgres-1                   | 
pulp-postgres-1                   | 
pulp-postgres-1                   | Success. You can now start the database server using:
pulp-postgres-1                   | 
pulp-postgres-1                   |     pg_ctl -D /var/lib/postgresql/data -l logfile start
pulp-postgres-1                   | 
pulp-postgres-1                   | waiting for server to start....2024-06-11 12:41:46.191 UTC [48] LOG:  starting PostgreSQL 13.15 (Debian 13.15-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit
pulp-postgres-1                   | 2024-06-11 12:41:46.192 UTC [48] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
pulp-postgres-1                   | 2024-06-11 12:41:46.216 UTC [49] LOG:  database system was shut down at 2024-06-11 12:41:46 UTC
pulp-postgres-1                   | 2024-06-11 12:41:46.221 UTC [48] LOG:  database system is ready to accept connections
pulp-postgres-1                   |  done
pulp-postgres-1                   | server started
pulp-postgres-1                   | CREATE DATABASE
pulp-postgres-1                   | 
pulp-postgres-1                   | 
pulp-postgres-1                   | /usr/local/bin/docker-entrypoint.sh: ignoring /docker-entrypoint-initdb.d/*
pulp-postgres-1                   | 
pulp-postgres-1                   | 2024-06-11 12:41:46.755 UTC [48] LOG:  received fast shutdown request
pulp-postgres-1                   | waiting for server to shut down....2024-06-11 12:41:46.756 UTC [48] LOG:  aborting any active transactions
pulp-postgres-1                   | 2024-06-11 12:41:46.758 UTC [48] LOG:  background worker "logical replication launcher" (PID 55) exited with exit code 1
pulp-postgres-1                   | 2024-06-11 12:41:46.759 UTC [50] LOG:  shutting down
pulp-postgres-1                   | 2024-06-11 12:41:46.770 UTC [48] LOG:  database system is shut down
pulp-postgres-1                   |  done
pulp-postgres-1                   | server stopped
pulp-postgres-1                   | 
pulp-postgres-1                   | PostgreSQL init process complete; ready for start up.
pulp-postgres-1                   | 
pulp-postgres-1                   | 2024-06-11 12:41:46.905 UTC [1] LOG:  starting PostgreSQL 13.15 (Debian 13.15-1.pgdg120+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 12.2.0-14) 12.2.0, 64-bit
pulp-postgres-1                   | 2024-06-11 12:41:46.905 UTC [1] LOG:  listening on IPv4 address "0.0.0.0", port 5432
pulp-postgres-1                   | 2024-06-11 12:41:46.905 UTC [1] LOG:  listening on IPv6 address "::", port 5432
pulp-postgres-1                   | 2024-06-11 12:41:46.907 UTC [1] LOG:  listening on Unix socket "/var/run/postgresql/.s.PGSQL.5432"
pulp-postgres-1                   | 2024-06-11 12:41:46.911 UTC [63] LOG:  database system was shut down at 2024-06-11 12:41:46 UTC
pulp-postgres-1                   | 2024-06-11 12:41:46.916 UTC [1] LOG:  database system is ready to accept connections
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.730 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128.
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.731 * Server initialized
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.753 * Loading RDB produced by version 7.2.5
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.753 * RDB age 214 seconds
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.753 * RDB memory usage when created 0.85 Mb
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.753 * Done loading RDB, keys loaded: 0, keys expired: 0.
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.753 * DB loaded from disk: 0.023 seconds
pulp-redis-1                      | 1:M 11 Jun 2024 12:41:44.753 * Ready to accept connections tcp
pulp-postgres-1                   | 2024-06-11 12:41:58.226 UTC [78] ERROR:  relation "core_artifact" does not exist at character 28
pulp-postgres-1                   | 2024-06-11 12:41:58.226 UTC [78] STATEMENT:  SELECT count(pulp_id) FROM core_artifact WHERE sha224 IS NULL
pulp-migration_service-1 exited with code 139
pulp-postgres-1                   | 2024-06-11 12:41:59.373 UTC [80] ERROR:  relation "core_artifact" does not exist at character 28
pulp-postgres-1                   | 2024-06-11 12:41:59.373 UTC [80] STATEMENT:  SELECT count(pulp_id) FROM core_artifact WHERE sha224 IS NULL
pulp-postgres-1                   | 2024-06-11 12:42:00.849 UTC [81] ERROR:  relation "core_artifact" does not exist at character 28
pulp-postgres-1                   | 2024-06-11 12:42:00.849 UTC [81] STATEMENT:  SELECT count(pulp_id) FROM core_artifact WHERE sha224 IS NULL
pulp-set_init_password_service-1 exited with code 134
mikedep333 commented 1 month ago

Hi @jeremyatourville This is surprising. We tried reproducing it, albiet with the latest images, and it worked. Our usual advise would be to drop the database and try again, although it looks like you did create the fresh database directory.

jeremyatourville commented 1 month ago

@mikedep333 When you say drop the database - is there a specific command to run? I am getting a message that you can't drop the currently open DB. Or are you saying delete everything under the pgsql folder?

Alternatively, what image versions were you able to make work? My understanding is that the container image tag latest is sort of a relative reference. It is the latest at the time of download but there may be changes if you download it at a later time. I also tried to modify my compose file and run pulp-minimal:3.54.0 and redis:7.2.5 This had no effect on the error. It still tells me ERROR: relation "core_artifact" does not exist at character 28