simulate test harbor warehouse object storage S3 failure, node cache image cannot be download

mansoncui commented 3 months ago

problem: simulate test harbor warehouse object storage S3 failure, node cache image cannot be download?

before closing minio s3, image can be download normally

root@ops-k8s-testwxnode8:~# /opt/kubernetes/bin/crictl pull  harbor.test.wifi.com/library/clickhouse-server:20.3.9.70
Image is up to date for sha256:abe55fc6544d42937af1eccad0a639a039bded091b0972124d3069eba82089b8

close minio s3

download image
## /opt/kubernetes/bin/crictl pull  harbor.test.wifi.com/library/clickhouse-server:20.3.9.70 
E0606 19:12:26.429217   30123 remote_image.go:238] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\": failed to resolve reference \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\": pulling from host 127.0.0.1:65001 failed with status code [manifests 20.3.9.70]: 500 Internal Server Error" image="harbor.test.wifi.com/library/clickhouse-server:20.3.9.70"
FATA[0048] pulling image: rpc error: code = Unknown desc = failed to pull and unpack image "harbor.test.wifi.com/library/clickhouse-server:20.3.9.70": failed to resolve reference "harbor.test.wifi.com/library/clickhouse-server:20.3.9.70": pulling from host 127.0.0.1:65001 failed with status code [manifests 20.3.9.70]: 500 Internal Server Error

scheduler core.log


{"level":"info","ts":"2024-06-06 10:57:05.004","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc000771ab0), BeginTime:0x17d6650f7601702f, EndTime:0x17d6650f783332f5, Success:true, Code:200, FinishedCount:18, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:27, RangeStart:0x6c00000, RangeSize:0x400000, PieceMd5:\"d034a86c17a0bf7bc11c4a91255579db\", PieceOffset:0x6c00000, PieceStyle:0, DownloadCost:0x7854}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.006","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.87.184-27447-66351329-ab97-46b0-a570-f25f70ebe0ba\", PieceInfo:(*common.PieceInfo)(0xc000771b90), BeginTime:0x17d6650f76ef7065, EndTime:0x17d6650f7858ef2c, Success:true, Code:200, FinishedCount:19, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:20, RangeStart:0x5000000, RangeSize:0x400000, PieceMd5:\"0d276af7efa73351b6eae501b1cd43a3\", PieceOffset:0x5000000, PieceStyle:0, DownloadCost:0x14d66}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.006","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc000771ce0), BeginTime:0x17d6650f760a8a1c, EndTime:0x17d6650f78589c10, Success:true, Code:200, FinishedCount:19, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:16, RangeStart:0x4000000, RangeSize:0x400000, PieceMd5:\"88c03447b108c0d5bcdec1c5a325050f\", PieceOffset:0x4000000, PieceStyle:0, DownloadCost:0x714c}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.011","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.30.40-19280-55e0c0a5-4222-48db-8a47-49a71449464c\", PieceInfo:(*common.PieceInfo)(0xc000771dc0), BeginTime:0x17d6650f77448431, EndTime:0x17d6650f78a2db16, Success:true, Code:200, FinishedCount:21, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:23, RangeStart:0x5c00000, RangeSize:0x400000, PieceMd5:\"4c7a7320f8bfe002ae45109fb0effa28\", PieceOffset:0x5c00000, PieceStyle:0, DownloadCost:0x17638}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.068","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc000771f10), BeginTime:0x17d6650f7834faff, EndTime:0x17d6650f7c01e6a6, Success:true, Code:200, FinishedCount:22, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:22, RangeStart:0x5800000, RangeSize:0x400000, PieceMd5:\"922a3f4ccdc6a8f0e56706557ddf711b\", PieceOffset:0x5800000, PieceStyle:0, DownloadCost:0x704e}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.071","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc00070e460), BeginTime:0x17d6650f785a4e2d, EndTime:0x17d6650f7c36d6df, Success:true, Code:200, FinishedCount:23, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:4, RangeStart:0x1000000, RangeSize:0x400000, PieceMd5:\"75de8b002f98510d975eaac52636ee0e\", PieceOffset:0x1000000, PieceStyle:0, DownloadCost:0x719d}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.074","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.79.36-1-3222b847-bf2f-46d2-ad14-9fdd4cea971d_Seed\", PieceInfo:(*common.PieceInfo)(0xc00070e620), BeginTime:0x17d6650f785aba3f, EndTime:0x17d6650f7c60e656, Success:true, Code:200, FinishedCount:24, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:9, RangeStart:0x2400000, RangeSize:0x400000, PieceMd5:\"8340b0b5f8a90d55c528a19e3e5acc4e\", PieceOffset:0x2400000, PieceStyle:0, DownloadCost:0x71d8}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}
{"level":"info","ts":"2024-06-06 10:57:05.077","caller":"service/service_v1.go:250","msg":"receive success piece: &scheduler.PieceResult{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(0xc000c0ca20)}, sizeCache:0, unknownFields:[]uint8(nil), TaskId:\"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de\", SrcPid:\"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e\", DstPid:\"10.222.30.40-19280-55e0c0a5-4222-48db-8a47-49a71449464c\", PieceInfo:(*common.PieceInfo)(0xc00070e7e0), BeginTime:0x17d6650f78a44cb1, EndTime:0x17d6650f7c8ee5a4, Success:true, Code:200, FinishedCount:25, ExtendAttribute:(*common.ExtendAttribute)(nil)} &common.PieceInfo{state:impl.MessageState{NoUnkeyedLiterals:pragma.NoUnkeyedLiterals{}, DoNotCompare:pragma.DoNotCompare{}, DoNotCopy:pragma.DoNotCopy{}, atomicMessageInfo:(*impl.MessageInfo)(nil)}, sizeCache:0, unknownFields:[]uint8(nil), PieceNum:10, RangeStart:0x2800000, RangeSize:0x400000, PieceMd5:\"63ade6173630f10623366a87aab09461\", PieceOffset:0x2800000, PieceStyle:0, DownloadCost:0x19c62}","hostID":"890e03447134f7e13b3dc548ed0af47ccb05e033c0309fa02a59cda5fb094b41","taskID":"4f56bd80e6963891142130a871173dfe1bed94cc01367f2c4f85caf0ef8c68de","peerID":"10.222.79.9-16589-e61c0a7d-c937-462f-99bd-cf539fa1a97e"}

seed peer core.log not log

helm install value.yaml 删除 injectConfigPath: true这个参数，会在/etc/containerd/config.toml文件新增以下配置

[plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.test.wifi.com"]
   endpoint = ["http://127.0.0.1:65001", "https://harbor.test.wifi.com"]

Dragonfly version: 2.1.0 OS: ubuntu 18.04 Kernel (e.g. uname -a): 5.4.193-0504193-generic Others: k8s 1.23.6 containerd 1.6.6

gaius-qi commented 3 months ago

@mansoncui Please provide entrie logs(Peer、Seed Peer、Scheduler)

mansoncui commented 3 months ago

@mansoncui Please provide entrie logs(Peer、Seed Peer、Scheduler)

peer log info：

{"level":"debug","ts":"2024-06-07 10:32:35.485","caller":"transport/transport.go:214","msg":"round trip directly, method: HEAD, url: https://harbor.test.wifi.com/v2/library/clickhouse-server/manifests/20.3.9.70?ns=harbor.test.wifi.com"}
{"level":"debug","ts":"2024-06-07 10:32:35.593","caller":"transport/transport.go:214","msg":"round trip directly, method: HEAD, url: https://harbor.test.wifi.com/v2/library/clickhouse-server/manifests/20.3.9.70?ns=harbor.test.wifi.com"}

Containerd info log:

Jun 07 18:32:35 ops-k8s-testwxnode11.wxxdc containerd[22110]: time="2024-06-07T18:32:35.484731691+08:00" level=info msg="PullImage \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\""
Jun 07 18:33:03 ops-k8s-testwxnode11.wxxdc containerd[22110]: time="2024-06-07T18:33:03.652154734+08:00" level=info msg="trying next host" error="failed to do request: Head \"https://harbor.test.wifi.com/v2/library/clickhouse-server/manifests/20.3.9.70\": x509: certificate signed by unknown authority" host=harbor.test.wifi.com
Jun 07 18:33:03 ops-k8s-testwxnode11.wxxdc containerd[22110]: time="2024-06-07T18:33:03.652581364+08:00" level=error msg="PullImage \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\" failed" error="failed to pull and unpack image \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\": failed to resolve reference \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\": pulling from host 127.0.0.1:65001 failed with status code [manifests 20.3.9.70]: 500 Internal Server Error"

scheduler and seed peer not logs

dfdaemon skip ssl

value.yaml

containerRuntime:
  containerd:
    enable: true
    registries:
      - 'https://harbor.test.wifi.com'

scheduler:
  replicas: 1
  metrics:
    enable: true
    serviceMonitor:
      enable: true
  config:
    verbose: true
    pprofPort: 18066
    scheduler:
      gc:
        peerTTL: 96h
        hostGCInterval: 72h

seedPeer:
  enable: true
  image:
    repository: dragonflyoss/dfdaemon
    tag: latest
  replicas: 1
  metrics:
    enable: true
    serviceMonitor:
      enable: true
  config:
    verbose: true
    pprofPort: 18066
    proxy:
      defaultFilter: 'Expires&Signature&ns'
      security:
        insecure: true
      tcpListen:
        listen: 0.0.0.0
        port: 65001
      registryMirror:
        dynamic: true
        url: https://harbor.test.wifi.com
        insecure: true
      proxies:
        - regx: blobs/sha256.*

dfdaemon:
  enable: true
  image:
    registry: harbor-wf.internal.zenmen.com
    repository: dragonflyoss/dfdaemon-certs-1
    tag: latest
  metrics:
    enable: true
    serviceMonitor:
      enable: true
  config:
    verbose: true
    pprofPort: 18066
    storage:
      taskExpireTime: 72h
      multiplex: false
    download:
      totalRateLimit: 4096Mi
      perPeerRateLimit: 4096Mi
      resourceClients:
        https:
          insecureSkipVerify: true
    upload:
      rateLimit: 4096Mi
    scheduler:
      manager:
        enable: true
        seedPeer:
          enable: true
          type: super
          clusterID: 1
          keepAlive:
            internal: 5s
    proxy:
      defaultFilter: 'Expires&Signature&ns'
      security:
        insecure: true
      tcpListen:
        listen: 0.0.0.0
        port: 65001
      registryMirror:
        dynamic: true
        url: https://harbor.test.wifi.com
        insecure: true
        certs: 
          - /etc/docker/certs.d/ca.crt
      proxies:
        - regx: blobs/sha256.*

manager:
  replicas: 1
  metrics:
    enable: true
    serviceMonitor:
      enable: true
  config:
    verbose: true
    pprofPort: 18066
    job:
      syncPeers:
        interval: 13h
        timeout: 10m
      preheat:
        registryTimeout: 1m
        tls:
          # caCert is the CA certificate for preheat tls handshake, it can be path or PEM format string.
          caCert: | 
            -----BEGIN CERTIFICATE-----

            -----END CERTIFICATE-----                  

jaeger:
  enable: true

@gaius-qi

gaius-qi commented 3 months ago

@mansoncui Please provide entrie logs(Peer、Seed Peer、Scheduler)

peer log info：


{"level":"debug","ts":"2024-06-07 10:32:35.485","caller":"transport/transport.go:214","msg":"round trip directly, method: HEAD, url: https://harbor.test.wifi.com/v2/library/clickhouse-server/manifests/20.3.9.70?ns=harbor.test.wifi.com"}

{"level":"debug","ts":"2024-06-07 10:32:35.593","caller":"transport/transport.go:214","msg":"round trip directly, method: HEAD, url: https://harbor.test.wifi.com/v2/library/clickhouse-server/manifests/20.3.9.70?ns=harbor.test.wifi.com"}

Containerd info log:


Jun 07 18:32:35 ops-k8s-testwxnode11.wxxdc containerd[22110]: time="2024-06-07T18:32:35.484731691+08:00" level=info msg="PullImage \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\""

Jun 07 18:33:03 ops-k8s-testwxnode11.wxxdc containerd[22110]: time="2024-06-07T18:33:03.652154734+08:00" level=info msg="trying next host" error="failed to do request: Head \"https://harbor.test.wifi.com/v2/library/clickhouse-server/manifests/20.3.9.70\": x509: certificate signed by unknown authority" host=harbor.test.wifi.com

Jun 07 18:33:03 ops-k8s-testwxnode11.wxxdc containerd[22110]: time="2024-06-07T18:33:03.652581364+08:00" level=error msg="PullImage \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\" failed" error="failed to pull and unpack image \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\": failed to resolve reference \"harbor.test.wifi.com/library/clickhouse-server:20.3.9.70\": pulling from host 127.0.0.1:65001 failed with status code [manifests 20.3.9.70]: 500 Internal Server Error"

scheduler and seed peer not logs

dfdaemon skip ssl

value.yaml


containerRuntime:

  containerd:

    enable: true

    registries:

      - 'https://harbor.test.wifi.com'

scheduler:

  replicas: 1

  metrics:

    enable: true

    serviceMonitor:

      enable: true

  config:

    verbose: true

    pprofPort: 18066

    scheduler:

      gc:

        peerTTL: 96h

        hostGCInterval: 72h

seedPeer:

  enable: true

  image:

    repository: dragonflyoss/dfdaemon

    tag: latest

  replicas: 1

  metrics:

    enable: true

    serviceMonitor:

      enable: true

  config:

    verbose: true

    pprofPort: 18066

    proxy:

      defaultFilter: 'Expires&Signature&ns'

      security:

        insecure: true

      tcpListen:

        listen: 0.0.0.0

        port: 65001

      registryMirror:

        dynamic: true

        url: https://harbor.test.wifi.com

        insecure: true

      proxies:

        - regx: blobs/sha256.*

dfdaemon:

  enable: true

  image:

    registry: harbor-wf.internal.zenmen.com

    repository: dragonflyoss/dfdaemon-certs-1

    tag: latest

  metrics:

    enable: true

    serviceMonitor:

      enable: true

  config:

    verbose: true

    pprofPort: 18066

    storage:

      taskExpireTime: 72h

      multiplex: false

    download:

      totalRateLimit: 4096Mi

      perPeerRateLimit: 4096Mi

      resourceClients:

        https:

          insecureSkipVerify: true

    upload:

      rateLimit: 4096Mi

    scheduler:

      manager:

        enable: true

        seedPeer:

          enable: true

          type: super

          clusterID: 1

          keepAlive:

            internal: 5s

    proxy:

      defaultFilter: 'Expires&Signature&ns'

      security:

        insecure: true

      tcpListen:

        listen: 0.0.0.0

        port: 65001

      registryMirror:

        dynamic: true

        url: https://harbor.test.wifi.com

        insecure: true

        certs: 

          - /etc/docker/certs.d/ca.crt

      proxies:

        - regx: blobs/sha256.*

manager:

  replicas: 1

  metrics:

    enable: true

    serviceMonitor:

      enable: true

  config:

    verbose: true

    pprofPort: 18066

    job:

      syncPeers:

        interval: 13h

        timeout: 10m

      preheat:

        registryTimeout: 1m

        tls:

          # caCert is the CA certificate for preheat tls handshake, it can be path or PEM format string.

          caCert: | 

            -----BEGIN CERTIFICATE-----

            -----END CERTIFICATE-----                  

jaeger:

  enable: true

@gaius-qi

@jim3ma

dragonflyoss / Dragonfly2

simulate test harbor warehouse object storage S3 failure, node cache image cannot be download #3310