Upgrade fails v3.1.1 to v3.3.0 with err=empty ring msg=error getting ingester clients

Describe the bug The upgrade from v3.1.1 to v3.3.0 fails with the following error message:

INF ring.go:316 > | component=distributor msg=ring doesn't exist in KV store yet
ERR ratestore.go:109 > | err=empty ring msg=error getting ingester clients ts=2024-11-20T07:28:06.085153173Z
WRN logging.go:128 > | msg=POST /loki/api/v1/push (500) 504.95µs Response: "empty ring\n" ws: false; Content-Length: 633; Content-Type: application/x-protobuf; User-Agent: Alloy/v1.5.0 (linux; rpm)

loki confg:

auth_enabled: false

analytics:
 reporting_enabled: false

server:
http_listen_port:3100
http_server_write_timeout:10m
http_server_read_timeout:10m
http_server_idle_timeout:10m
graceful_shutdown_timeout:2m
grpc_listen_port:9095
grpc_server_max_recv_msg_size:838866080
grpc_server_max_send_msg_size:838866080
log_level:info
log_format:json

common:
compactor_address:http://loki:3100
path_prefix:/loki
replication_factor:1
  storage:
    s3:
bucketnames:loki-data
      endpoint: ${LOKI_S3_ENDPOINT:-minio:9000}
      access_key_id: ${LOKI_S3_ACCESS_KEY_ID:-lgtmp}
      secret_access_key: ${LOKI_S3_SECRET_ACCESS_KEY:-supersecret}
      insecure: ${LOKI_S3_INSECURE:-true}
      s3forcepathstyle: true
      http_config:
        insecure_skip_verify: true

bloom_gateway:
  enabled: true
  client:
    addresses: "dns+loki:9095"
    cache_results: true
    results_cache:
      cache:
        memcached_client:
          addresses: "dns+memcached:11211"

index_gateway:
mode:simple

compactor:
working_directory:/tmp/compactor

memberlist:
  join_members:
  -loki-memberlist:7946

# https://github.com/grafana/loki/blob/main/docs/sources/configure/_index.md#query_range
query_range:
  align_queries_with_step: true

  cache_results: true
  results_cache:
    cache:
      memcached_client:
        addresses: "dns+memcached:11211"

  cache_index_stats_results: true
  index_stats_results_cache:
    cache:
      memcached_client:
        addresses: "dns+memcached:11211"
  # cache_volume_results: true
  # cache_series_results: true
  # instant_metric_query_split_align: true
  # cache_instant_metric_results: true
  # instant_metric_results_cache:
  #   cache:
  #     memcached_client:
  #       addresses: "dns+memcached:11211"
  # series_results_cache:
  #   cache:
  #     memcached_client:
  #       addresses: dns+memcached:11211"
  # volume_results_cache:
  #   cache:
  #     memcached_client:
  #       addresses: "dns+memcached:11211"

pattern_ingester:
  enabled: true

limits_config:
max_global_streams_per_user:0
ingestion_rate_mb:50000
ingestion_burst_size_mb:50000
  volume_enabled: true
  allow_structured_metadata: true
  reject_old_samples: false
retention_period:672h
reject_old_samples_max_age:672h
split_queries_by_interval:24h
max_query_parallelism:32

ruler:
  storage:
    s3:
bucketnames:loki-ruler
type:s3

runtime_config:
file:/etc/loki/configs/runtime.yaml

schema_config:
  configs:
  - from: "2024-10-24"
    index:
prefix:loki_index_
period:24h
object_store:s3
store:tsdb
schema:v13

storage_config:
  tsdb_shipper:
active_index_directory:/loki/index
cache_location:/loki/cache
    index_gateway_client:
      server_address: "dns+loki:9095"

chunk_store_config:
  chunk_cache_config:
    memcached_client:
      addresses: "dns+memcached:11211"

To Reproduce Steps to reproduce the behavior: Upgrade attempt v3.3.3 with this configuration

Expected behavior Flawless upgrade to v3.3.3

grafana / loki

Upgrade fails v3.1.1 to v3.3.0 with err=empty ring msg=error getting ingester clients #15030