Open san3Xian opened 1 month ago
redhat 8.5 with later kernel cant not build the drbd module in drbd-adapter job
kubectl get deployments.apps -owide -n hwameistor hwameistor-operator NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR hwameistor-operator 1/1 1 1 36d hwameistor-operator 10.101.210.43/ghcr.m.daocloud.io/hwameistor/operator:v0.14.9 app=hwameistor-operator
log of shipper container:
kubectl logs -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -c shipper '/files/Dockerfile.shipper' -> '/pkgs/Dockerfile.shipper' '/files/drbd-debs' -> '/pkgs/drbd-debs' '/files/drbd-reactor.toml' -> '/pkgs/drbd-reactor.toml' '/files/drbd-rpms' -> '/pkgs/drbd-rpms' '/files/drbd.conf' -> '/pkgs/drbd.conf' '/files/drbd.modules' -> '/pkgs/drbd.modules' '/files/drbd.modules-load.conf' -> '/pkgs/drbd.modules-load.conf' '/files/entrypoint.adapter.sh' -> '/pkgs/entrypoint.adapter.sh' '/files/global_common.conf' -> '/pkgs/global_common.conf' '/files/install_opam.sh' -> '/pkgs/install_opam.sh' '/files/kernel-devels.centos7' -> '/pkgs/kernel-devels.centos7' '/files/kernel-devels.centos8' -> '/pkgs/kernel-devels.centos8' '/files/rhel7.0' -> '/pkgs/rhel7.0' '/files/rhel7.1' -> '/pkgs/rhel7.1' '/files/rhel7.2' -> '/pkgs/rhel7.2' '/files/rhel7.3' -> '/pkgs/rhel7.3' '/files/rhel7.4' -> '/pkgs/rhel7.4' '/files/rhel7.5' -> '/pkgs/rhel7.5' '/files/rhel7.6' -> '/pkgs/rhel7.6' '/files/rhel7.7' -> '/pkgs/rhel7.7' '/files/rhel7.8' -> '/pkgs/rhel7.8' '/files/rhel7.9' -> '/pkgs/rhel7.9' '/files/rhel8.0' -> '/pkgs/rhel8.0' '/files/rhel8.1' -> '/pkgs/rhel8.1' '/files/rhel8.2' -> '/pkgs/rhel8.2' '/files/rhel8.3' -> '/pkgs/rhel8.3' '/files/rhel8.4' -> '/pkgs/rhel8.4' '/files/rhel8.5' -> '/pkgs/rhel8.5' '/files/rhel8.6' -> '/pkgs/rhel8.6' '/files/utils' -> '/pkgs/utils'
log of rhel8 container:
[root@mitqadaocappn02 ~]# kubectl logs -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -c rhel8 + which lbdisttool.py /usr/local/bin/lbdisttool.py ++ lbdisttool.py -l ++ awk -F. '{print $1}' + image_dist=rhel8 ++ lbdisttool.py -l --os-release /etc/host-release ++ awk -F. '{print $1}' + host_dist=rhel8 + '[' -z rhel8 ']' + '[' -z rhel8 ']' + [[ rhel8 != rhel8 ]] ++ cat /proc/drbd ++ awk '/^version:/ {print $2}' cat: /proc/drbd: No such file or directory + RUNNING_DRBD_VERSION= + '[' -z ']' + echo 'No DRBD Module is loaded' + echo OS_KERNEL:4.18.0-477.10.1.el8_8.x86_64 + export DRBD_EXIST=no + DRBD_EXIST=no No DRBD Module is loaded OS_KERNEL:4.18.0-477.10.1.el8_8.x86_64 + export DRBD_RMP_INSTALL=no + DRBD_RMP_INSTALL=no + export OS_KERNEL=4.18.0-477.10.1.el8_8.x86_64 + OS_KERNEL=4.18.0-477.10.1.el8_8.x86_64 + export OS_HOST_DIST=rhel8 + OS_HOST_DIST=rhel8 + [[ rhel8 == \k\y\l\i\n\1\0 ]] + LB_HOW=shipped_modules + [[ shipped_modules == \s\h\i\p\p\e\d\_\m\o\d\u\l\e\s ]] + bash -x /entry.sh + SIGN_KEY=https://packages.linbit.com/package-signing-pubkey.asc + PKGS=/pkgs + HOSTRELEASE=/etc/host-release + RPMDIR=/pkgs/drbd-rpms/ + DEBDIR=/pkgs/drbd-debs/ + HOW_DEPSONLY=deps_only + HOW_REPOFILE=repo_file + HOW_HASH=node_hash + HOW_FROMSRC=compile + HOW_FROMSHIPPED=shipped_modules + HOW_LOAD_FROM_RAM=RAM + HOW_INSTALL=install + modprobe_deps + local s + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe libcrc32c + s=success + debug 'Loading libcrc32c: success' + '[' -n '' ']' + return 0 + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe nvmet_rdma + s=success + debug 'Loading nvmet_rdma: success' + '[' -n '' ']' + return 0 + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe nvme_rdma + s=success + debug 'Loading nvme_rdma: success' + '[' -n '' ']' + return 0 + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe loop + s=success + debug 'Loading loop: success' + '[' -n '' ']' + return 0 + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe dm_writecache + s=success + debug 'Loading dm_writecache: success' + '[' -n '' ']' + return 0 + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe dm_cache + s=success + debug 'Loading dm_cache: success' + '[' -n '' ']' + return 0 + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe dm_thin_pool + s=success + debug 'Loading dm_thin_pool: success' + '[' -n '' ']' + return 0 + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe dm_snapshot + s=success + debug 'Loading dm_snapshot: success' + '[' -n '' ']' + return 0 + for m in libcrc32c nvmet_rdma nvme_rdma loop dm_writecache dm_cache dm_thin_pool dm_snapshot dm_crypt + modprobe dm_crypt + s=success + debug 'Loading dm_crypt: success' + '[' -n '' ']' + return 0 + return 0 + [[ '' == \d\e\p\s\_\o\n\l\y ]] + grep -q '^drbd ' /proc/modules + '[' -z 4.18.0-477.10.1.el8_8.x86_64 ']' + var1=4.18.0-477.10.1.el8_8.x86_64 + matchvar=4.18.0-477.10.1.el8_8.x86_64.rpm ++ find /pkgs/drbd-rpms/ -type f -name '*4.18.0-477.10.1.el8_8.x86_64.rpm' -print -quit + drbd_rpm= + '[' -n '' ']' + matchvar=4.18.0-477.10.1.el8_8.x86_64.deb ++ find /pkgs/drbd-debs/ -type f -name '*4.18.0-477.10.1.el8_8.x86_64.deb' -print -quit + drbd_deb= + [[ -n '' ]] + debug 'There is no corresponding kernel version deb package' + '[' -n '' ']' + return 0 + pkgdir=/tmp/pkg + kodir=/tmp/ko + rm -rf /tmp/pkg /tmp/ko + mkdir -p /tmp/pkg /tmp/ko + fmt=rpm ++ type -p dpkg + '[' -n '' ']' ++ type -p emerge + '[' -n '' ']' + case $fmt in ++ repo::rpm::getrepofile ++ echo /etc/yum.repos.d/linbit.repo + repo=/etc/yum.repos.d/linbit.repo ++ how_to_get /etc/yum.repos.d/linbit.repo ++ local repo=/etc/yum.repos.d/linbit.repo ++ local how= ++ '[' -n '' ']' ++ '[' -f /etc/yum.repos.d/linbit.repo ']' ++ '[' -n '' ']' ++ mountpoint -q /usr/src ++ how=compile ++ echo compile ++ return 0 + how_get=compile + debug 'Detected kmod method: "compile"' + '[' -n '' ']' + return 0 + dist=we_do_not_care ++ needs_dist compile ++ local how=compile ++ local needsdist=n ++ [[ compile == \n\o\d\e\_\h\a\s\h ]] ++ echo n ++ return 0 + need_dist=n + debug 'Needs host distribution info: "n"' + '[' -n '' ']' + return 0 + [[ n == y ]] + case $how_get in + kos::fromsrc /tmp/pkg /tmp/ko + local pkgdir=/tmp/pkg + local kodir=/tmp/ko + cd /tmp/pkg + tar xf /drbd.tar.gz + cd drbd-9.0.32-1 + make Need a git checkout to regenerate drbd/.drbd_git_revision make[1]: Entering directory '/tmp/pkg/drbd-9.0.32-1/drbd' Calling toplevel makefile of kernel source tree, which I believe is in KDIR=/lib/modules/4.18.0-477.10.1.el8_8.x86_64/build make -C /lib/modules/4.18.0-477.10.1.el8_8.x86_64/build M=/tmp/pkg/drbd-9.0.32-1/drbd modules Makefile:1006: *** "Cannot generate ORC metadata for CONFIG_UNWINDER_ORC=y, please install libelf-dev, libelf-devel or elfutils-libelf-devel". Stop. make[1]: Leaving directory '/tmp/pkg/drbd-9.0.32-1/drbd' make[1]: *** [Makefile:132: kbuild] Error 2 make: *** [Makefile:131: module] Error 2 + [[ rhel8 == \k\y\l\i\n\1\0 ]] ++ how_to_load ++ [[ '' == yes ]] ++ echo RAM ++ return 0 + how_load=RAM + debug 'Detected load method: "RAM"' + '[' -n '' ']' + return 0 + [[ compile == \c\o\m\p\i\l\e ]] + [[ RAM == \i\n\s\t\a\l\l ]] + load_from_ram /tmp/pkg /tmp/ko + local pkgdir=/tmp/pkg + local kodir=/tmp/ko + find /tmp/pkg -name '*.ko' -exec cp '{}' /tmp/ko ';' + cd /tmp/ko + '[' '!' -f drbd.ko ']' + die 'Could not find the expexted *.ko, see stderr for more details' + echo + echo -e 'Could not find the expexted *.ko, see stderr for more details' Could not find the expexted *.ko, see stderr for more details + exit 1 Successfully loaded shipped module + echo 'Successfully loaded shipped module' DRBD IS ALREADY EXIST (yes or no):no DRBD INSTALL BY PRM (yes or no) :no + echo 'DRBD IS ALREADY EXIST (yes or no):no' + echo 'DRBD INSTALL BY PRM (yes or no) :no' + [[ no == \y\e\s ]] + [[ yes == yes ]] + [[ rhel8 =~ rhel|kylin10 ]] ++ uname -r + KODIR=/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd + mkdir -vp /lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd mkdir: created directory '/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra' mkdir: created directory '/lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd' + cp -vf '/tmp/ko/*.ko' /lib/modules/4.18.0-477.10.1.el8_8.x86_64/extra/drbd/ cp: cannot stat '/tmp/ko/*.ko': No such file or directory + depmod -a + cp -vf /pkgs/drbd.modules-load.conf /etc/modules-load.d/drbd.conf '/pkgs/drbd.modules-load.conf' -> '/etc/modules-load.d/drbd.conf' + cp -vf /pkgs/drbd.modules /etc/sysconfig/modules/ '/pkgs/drbd.modules' -> '/etc/sysconfig/modules/drbd.modules' + cp -vf /pkgs/utils/drbdadm /pkgs/utils/drbdadm-83 /pkgs/utils/drbdadm-84 /pkgs/utils/drbdmeta /pkgs/utils/drbdsetup /pkgs/utils/drbdsetup-83 /pkgs/utils/drbdsetup-84 /usr-local-bin '/pkgs/utils/drbdadm' -> '/usr-local-bin/drbdadm' '/pkgs/utils/drbdadm-83' -> '/usr-local-bin/drbdadm-83' '/pkgs/utils/drbdadm-84' -> '/usr-local-bin/drbdadm-84' '/pkgs/utils/drbdmeta' -> '/usr-local-bin/drbdmeta' '/pkgs/utils/drbdsetup' -> '/usr-local-bin/drbdsetup' '/pkgs/utils/drbdsetup-83' -> '/usr-local-bin/drbdsetup-83' '/pkgs/utils/drbdsetup-84' -> '/usr-local-bin/drbdsetup-84' + cat /pkgs/drbd.conf + cp -vf /pkgs/global_common.conf /etc/drbd.d/ '/pkgs/global_common.conf' -> '/etc/drbd.d/global_common.conf' ++ cat /proc/drbd ++ awk '/^version/ {print $2}' cat: /proc/drbd: No such file or directory + [[ '' != 9.0.32-1 ]] + echo 'ERROR: DRBD is NOT loaded with the right version' + exit 1 ERROR: DRBD is NOT loaded with the right version [root@hostname ~]#
pod yaml:
[root@mitqadaocappn02 ~]# kubectl get pods -n hwameistor drbd-adapter-hostname-rhel8-1725963185-mn2n2 -oyaml apiVersion: v1 kind: Pod metadata: creationTimestamp: "2024-09-10T10:13:06Z" generateName: drbd-adapter-hostname-rhel8-1725963185- labels: app: drbd-adapter batch.kubernetes.io/controller-uid: ebe37506-8320-4c60-afff-cd838643c1d9 batch.kubernetes.io/job-name: drbd-adapter-hostname-rhel8-1725963185 controller-uid: ebe37506-8320-4c60-afff-cd838643c1d9 drbd-version: v9.0.32-1 job-name: drbd-adapter-hostname-rhel8-1725963185 name: drbd-adapter-hostname-rhel8-1725963185-mn2n2 namespace: hwameistor ownerReferences: - apiVersion: batch/v1 blockOwnerDeletion: true controller: true kind: Job name: drbd-adapter-hostname-rhel8-1725963185 uid: ebe37506-8320-4c60-afff-cd838643c1d9 resourceVersion: "64000555" uid: 4e91e3e9-f5ef-4fef-aa3a-690cd2d21648 spec: containers: - image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-shipper:v9.0.32-1_v0.4.2 imagePullPolicy: IfNotPresent name: shipper resources: {} terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /pkgs name: pkgs - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access-xvc27 readOnly: true - command: - /pkgs/entrypoint.adapter.sh - 4.18.0-477.10.1.el8_8.x86_64 env: - name: LB_SKIP value: "no" - name: LB_DROP value: "yes" - name: LB_UPGRADE value: "no" - name: LB_CHECK_HOSTNAME value: "yes" - name: NODE_NAME valueFrom: fieldRef: apiVersion: v1 fieldPath: spec.nodeName image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-rhel8:v9.0.32-1_v0.4.2 imagePullPolicy: IfNotPresent name: rhel8 resources: {} securityContext: privileged: true terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /pkgs name: pkgs - mountPath: /pkgs_root name: pkgroot - mountPath: /etc/host-release name: os-release readOnly: true - mountPath: /usr/src name: usr-src readOnly: true - mountPath: /lib/modules name: lib-modules - mountPath: /usr-local-bin name: usr-local-bin - mountPath: /etc/drbd.conf name: etc-drbd-conf - mountPath: /etc/drbd.d name: etc-drbd-d - mountPath: /var/lib/drbd name: var-lib-drbd readOnly: true - mountPath: /etc/modules-load.d name: etc-modules-load - mountPath: /etc/sysconfig/modules name: etc-sysconfig-modules - mountPath: /etc/centos-release name: centos-release readOnly: true - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access-xvc27 readOnly: true dnsPolicy: ClusterFirst enableServiceLinks: true hostNetwork: true hostPID: true nodeName: hostname nodeSelector: kubernetes.io/hostname: hostname preemptionPolicy: PreemptLowerPriority priority: 0 restartPolicy: Never schedulerName: default-scheduler securityContext: {} serviceAccount: default serviceAccountName: default terminationGracePeriodSeconds: 0 tolerations: - effect: NoSchedule key: node-role.kubernetes.io/master operator: Exists - effect: NoSchedule key: node-role.kubernetes.io/control-plane operator: Exists - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 300 - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 300 volumes: - emptyDir: {} name: pkgs - hostPath: path: /root type: "" name: pkgroot - hostPath: path: /etc/os-release type: FileOrCreate name: os-release - hostPath: path: /etc/centos-release type: FileOrCreate name: centos-release - hostPath: path: /usr/src type: "" name: usr-src - hostPath: path: /lib/modules type: "" name: lib-modules - hostPath: path: /usr/local/bin type: "" name: usr-local-bin - hostPath: path: /etc/drbd.conf type: FileOrCreate name: etc-drbd-conf - hostPath: path: /etc/drbd.d type: DirectoryOrCreate name: etc-drbd-d - hostPath: path: /var/lib/drbd type: DirectoryOrCreate name: var-lib-drbd - hostPath: path: /etc/modules-load.d type: DirectoryOrCreate name: etc-modules-load - hostPath: path: /etc/sysconfig/modules type: DirectoryOrCreate name: etc-sysconfig-modules - name: kube-api-access-xvc27 projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3607 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace status: conditions: - lastProbeTime: null lastTransitionTime: "2024-09-10T10:19:37Z" status: "False" type: PodReadyToStartContainers - lastProbeTime: null lastTransitionTime: "2024-09-10T10:13:06Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2024-09-10T10:13:06Z" reason: PodFailed status: "False" type: Ready - lastProbeTime: null lastTransitionTime: "2024-09-10T10:13:06Z" reason: PodFailed status: "False" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2024-09-10T10:13:06Z" status: "True" type: PodScheduled containerStatuses: - containerID: containerd://db23f500e40fd1990ab6d2f116283d76d3dea9254c1c79d4b0926382f9211ff0 image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-rhel8:v9.0.32-1_v0.4.2 imageID: sha256:4ed5663ed4e00011ace7a6b11c846c4e45d3f1fe81720fe1b94f7f06fcf68ef9 lastState: {} name: rhel8 ready: false restartCount: 0 started: false state: terminated: containerID: containerd://db23f500e40fd1990ab6d2f116283d76d3dea9254c1c79d4b0926382f9211ff0 exitCode: 1 finishedAt: "2024-09-10T10:19:36Z" reason: Error startedAt: "2024-09-10T10:19:31Z" - containerID: containerd://d2ec646dfe9de29d9346af986b4b54a8bc0ee2fff81f00948768ff8e84ae839f image: 10.101.210.43/ghcr.m.daocloud.io/hwameistor/drbd9-shipper:v9.0.32-1_v0.4.2 imageID: sha256:d48131e7df28b10c410cacb77667f22c5bde6995015bc04e1a3a8c362f210388 lastState: {} name: shipper ready: false restartCount: 0 started: false state: terminated: containerID: containerd://d2ec646dfe9de29d9346af986b4b54a8bc0ee2fff81f00948768ff8e84ae839f exitCode: 0 finishedAt: "2024-09-10T10:13:07Z" reason: Completed startedAt: "2024-09-10T10:13:06Z" hostIP: 10.101.210.44 hostIPs: - ip: 10.101.210.44 phase: Failed podIP: 10.101.210.44 podIPs: - ip: 10.101.210.44 qosClass: BestEffort startTime: "2024-09-10T10:13:06Z" [root@hostname ~]#
peng9808
commented
1 month ago peng9808
commented
1 month ago
redhat 8.5 with later kernel cant not build the drbd module in drbd-adapter job
log of shipper container:
log of rhel8 container:
pod yaml: