search

kubernetes-sigs / kwok

Kubernetes WithOut Kubelet - Simulates thousands of Nodes and Clusters.
https://kwok.sigs.k8s.io
Apache License 2.0
2.54k stars 200 forks source link

Kwok in cluster kustomize failed #270

Closed qihuang0 closed 1 year ago

qihuang0 commented 1 year ago

How to use it?

What happened?

I want to install kwok in cluster in below way, image and report the issue: image

What did you expect to happen?

I want to use kwok in cluster

How can we reproduce it (as minimally and precisely as possible)?

。。

Anything else we need to know?

No response

OS version

On Centos:


</details>
wzshiming commented 1 year ago

Do you not seem to be following the steps?

https://kwok.sigs.k8s.io/docs/user/kwok-in-cluster/#render-kustomization-yaml

kubectl kustomize "${KWOK_WORK_DIR}" > "${KWOK_WORK_DIR}/kwok.yaml"
wzshiming commented 1 year ago 
➜  kwok git:(main) ✗ # Temporary directory
KWOK_WORK_DIR=$(mktemp -d)
# Kwok repository
KWOK_REPO=kubernetes-sigs/kwok
# Get latest
KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 27264    0 27264    0     0  37945      0 --:--:-- --:--:-- --:--:-- 39228
➜  kwok git:(main) ✗ 
➜  kwok git:(main) ✗ cat <<EOF > "${KWOK_WORK_DIR}/kustomization.yaml"
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
images:
  - name: registry.k8s.io/kwok/kwok
    newTag: "${KWOK_LATEST_RELEASE}"
resources:
  - "https://github.com/${KWOK_REPO}/kustomize/kwok?ref=${KWOK_LATEST_RELEASE}"
EOF
➜  kwok git:(main) ✗ kubectl kustomize "${KWOK_WORK_DIR}" > "${KWOK_WORK_DIR}/kwok.yaml"
➜  kwok git:(main) ✗ cat "${KWOK_WORK_DIR}/kwok.yaml"
apiVersion: v1
kind: ServiceAccount
metadata:
  labels:
    app: kwok-controller
  name: kwok-controller
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    app: kwok-controller
  name: kwok-controller
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - watch
  - list
  - get
- apiGroups:
  - ""
  resources:
  - nodes/status
  verbs:
  - update
  - patch
- apiGroups:
  - ""
  resources:
  - pods
  verbs:
  - watch
  - list
  - delete
  - update
  - patch
- apiGroups:
  - ""
  resources:
  - pods/status
  verbs:
  - update
  - patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  labels:
    app: kwok-controller
  name: kwok-controller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kwok-controller
subjects:
- kind: ServiceAccount
  name: kwok-controller
  namespace: kube-system
---
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: kwok-controller
  name: kwok-controller
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: kwok-controller
  template:
    metadata:
      labels:
        app: kwok-controller
      name: kwok-controller
      namespace: kube-system
    spec:
      containers:
      - args:
        - --manage-all-nodes=false
        - --manage-nodes-with-annotation-selector=kwok.x-k8s.io/node=fake
        - --manage-nodes-with-label-selector=
        - --disregard-status-with-annotation-selector=kwok.x-k8s.io/status=custom
        - --disregard-status-with-label-selector=
        - --node-ip=$(POD_IP)
        - --node-port=10247
        - --cidr=10.0.0.1/24
        env:
        - name: POD_IP
          valueFrom:
            fieldRef:
              fieldPath: status.podIP
        image: registry.k8s.io/kwok/kwok:v0.1.0
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 10247
            scheme: HTTP
          initialDelaySeconds: 2
          periodSeconds: 10
          timeoutSeconds: 2
        name: kwok-controller
        readinessProbe:
          failureThreshold: 5
          httpGet:
            path: /healthz
            port: 10247
            scheme: HTTP
          initialDelaySeconds: 2
          periodSeconds: 20
          timeoutSeconds: 2
      restartPolicy: Always
      serviceAccount: kwok-controller
      serviceAccountName: kwok-controller
qihuang0 commented 1 year ago 
[root@test kwok]# KWOK_WORK_DIR=$(mktemp -d)
[root@test kwok]# KWOK_REPO=kubernetes-sigs/kwok
[root@test kwok]# KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100 27264  100 27264    0     0  30916      0 --:--:-- --:--:-- --:--:-- 30911
[root@test kwok]# cat <<EOF > "${KWOK_WORK_DIR}/kustomization.yaml"
> apiVersion: kustomize.config.k8s.io/v1beta1
> kind: Kustomization
> images:
>   - name: registry.k8s.io/kwok/kwok
>     newTag: "${KWOK_LATEST_RELEASE}"
> resources:
>   - "https://github.com/${KWOK_REPO}/kustomize/kwok?ref=${KWOK_LATEST_RELEASE}"
> EOF

[root@test kwok]# kubectl kustomize "${KWOK_WORK_DIR}" > "${KWOK_WORK_DIR}/kwok.yaml"
Error: rawResources failed to read Resources: Load from path https://github.com/kubernetes-sigs/kwok/kustomize/kwok?ref=v0.1.0 failed: evalsymlink failure on '/tmp/tmp.YCbPX4dQyD/https:/github.com/kubernetes-sigs/kwok/kustomize/kwok?ref=v0.1.0' : lstat /tmp/tmp.YCbPX4dQyD/https:: no such file or directory

[root@test kwok]# cat "${KWOK_WORK_DIR}/kwok.yaml"
wzshiming commented 1 year ago

It seems that the version of kustomize does not support fetching resources from HTTP, what's the version?

qihuang0 commented 1 year ago 
kubectl version
Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.15", GitCommit:"58178e7f7aab455bc8de88d3bdd314b64141e7ee", GitTreeState:"clean", BuildDate:"2021-09-27T13:35:39Z", GoVersion:"go1.15.15", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.15", GitCommit:"58178e7f7aab455bc8de88d3bdd314b64141e7ee", GitTreeState:"clean", BuildDate:"2021-09-27T13:35:39Z", GoVersion:"go1.15.15", Compiler:"gc", Platform:"linux/amd64"}
wzshiming commented 1 year ago

I took a closer look at the error logs, and I think it's a bug in kustomize where the path is only half of the path when it tries to get the link

lstat /tmp/tmp.YCbPX4dQyD/https:: no such file or directory

qihuang0 commented 1 year ago

It seems that the version of kustomize does not support fetching resources from HTTP, what's the version?

Kubectl version | Kustomize version -- | -- < v1.14 | n/a v1.14-v1.20 | v2.0.3 v1.21 | v4.0.5 v1.22 | v4.2.0
qihuang0 commented 1 year ago

So we must update kubectl to solve it ?👀

wzshiming commented 1 year ago

Ah. There are other options, download the new kustomize, or just use the yaml I posted to you above.

qihuang0 commented 1 year ago

Ok, thanks