search

k3s-io / k3s

Lightweight Kubernetes
https://k3s.io
Apache License 2.0
27.01k stars 2.28k forks source link

[Release-1.27] - K3s uses incorrect wasm shims names #9652

Closed brandond closed 4 months ago

brandond commented 4 months ago

Backport fix for K3s uses incorrect wasm shims names

VestigeJ commented 4 months ago

Waiting on backports but existing behavior is as follows

$ sudo cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml

# File generated by k3s. DO NOT EDIT. Use config.toml.tmpl instead.
version = 2

[plugins."io.containerd.internal.v1.opt"]
  path = "/var/lib/rancher/k3s/agent/containerd"
[plugins."io.containerd.grpc.v1.cri"]
  stream_server_address = "127.0.0.1"
  stream_server_port = "10010"
  enable_selinux = false
  enable_unprivileged_ports = true
  enable_unprivileged_icmp = true
  sandbox_image = "rancher/mirrored-pause:3.6"

[plugins."io.containerd.grpc.v1.cri".containerd]
  snapshotter = "overlayfs"
  disable_snapshot_annotations = true

[plugins."io.containerd.grpc.v1.cri".cni]
  bin_dir = "/var/lib/rancher/k3s/data/a3b46c0299091b71bfcc617b1e1fec1845c13bdd848584ceb39d2e700e702a4b/bin"
  conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  runtime_type = "io.containerd.runc.v2"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
  SystemdCgroup = true

[plugins."io.containerd.grpc.v1.cri".registry]
  config_path = "/var/lib/rancher/k3s/agent/etc/containerd/certs.d"

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."lunatic"]
  runtime_type = "io.containerd.lunatic.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."lunatic".options]
  BinaryName = "/usr/local/bin/containerd-shim-lunatic-v1"
  SystemdCgroup = true

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."slight"]
  runtime_type = "io.containerd.slight.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."slight".options]
  BinaryName = "/usr/local/bin/containerd-shim-slight-v1"
  SystemdCgroup = true

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."wws"]
  runtime_type = "io.containerd.wws.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."wws".options]
  BinaryName = "/usr/local/bin/containerd-shim-wws-v1"
  SystemdCgroup = true

$ ls /usr/local/bin/

containerd-shim-lunatic-v1  containerd-shim-slight-v1  containerd-shim-spin-v2  containerd-shim-wws-v1  crictl  ctr  k3s  k3s-killall.sh  k3s-uninstall.sh  kubectl
VestigeJ commented 4 months ago

Environment Details

Reproduced using VERSION=v1.27.11+k3s1 Validated using COMMIT=1268cf0a02e509071ab8843129b8561aed3a1ffd

Infrastructure

Node(s) CPU architecture, OS, and version:

Linux 5.14.21-150500.53-default x86_64 GNU/Linux PRETTY_NAME="SUSE Linux Enterprise Server 15 SP5"

Cluster Configuration:

NAME               STATUS   ROLES                  AGE     VERSION
ip-1-1-6-233       Ready    control-plane,master   3m47s   v1.27.11+k3s-1268cf0a

Reproduction

``` $ curl https://get.k3s.io --output install-"k3s".sh $ sudo chmod +x install-"k3s".sh $ sudo groupadd --system etcd && sudo useradd -s /sbin/nologin --system -g etcd etcd $ sudo modprobe ip_vs_rr $ sudo modprobe ip_vs_wrr $ sudo modprobe ip_vs_sh $ sudo printf "on_oovm.panic_on_oom=0 \nvm.overcommit_memory=1 \nkernel.panic=10 \nkernel.panic_ps=1 \nkernel.panic_on_oops=1 \n" > ~/90-kubelet.conf $ sudo cp 90-kubelet.conf /etc/sysctl.d/ $ sudo systemctl restart systemd-sysctl $ sudo INSTALL_K3S_COMMIT=$COMMIT INSTALL_K3S_EXEC=server ./install-k3s.sh $ sudo cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml $ set_kubefig $ kgp -A $ ls $ k apply -f deploy.yaml -f spin-shim-deploy.yaml -f middleware.yaml $ export PUBLIC_IP=$(kubectl get ingress wasm-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}') $ curl -v http://$PUBLIC_IP/slight/hello $ curl -v http://$PUBLIC_IP/spin/go-hello $ curl -v http://$PUBLIC_IP/spin/hello" ``` **Results:** $ sudo cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml ``` # File generated by k3s. DO NOT EDIT. Use config.toml.tmpl instead. version = 2 [plugins."io.containerd.internal.v1.opt"] path = "/var/lib/rancher/k3s/agent/containerd" [plugins."io.containerd.grpc.v1.cri"] stream_server_address = "127.0.0.1" stream_server_port = "10010" enable_selinux = false enable_unprivileged_ports = true enable_unprivileged_icmp = true sandbox_image = "rancher/mirrored-pause:3.6" [plugins."io.containerd.grpc.v1.cri".containerd] snapshotter = "overlayfs" disable_snapshot_annotations = true [plugins."io.containerd.grpc.v1.cri".cni] bin_dir = "/var/lib/rancher/k3s/data/13047d5dd5d62ef632c35b9178b6040e2a103eb058888853dd769df9a278e8d3/bin" conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] runtime_type = "io.containerd.runc.v2" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] SystemdCgroup = true [plugins."io.containerd.grpc.v1.cri".registry] config_path = "/var/lib/rancher/k3s/agent/etc/containerd/certs.d" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."lunatic"] runtime_type = "io.containerd.lunatic.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."lunatic".options] BinaryName = "/usr/local/bin/containerd-shim-lunatic-v1" SystemdCgroup = true [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."slight"] runtime_type = "io.containerd.slight.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."slight".options] BinaryName = "/usr/local/bin/containerd-shim-slight-v1" SystemdCgroup = true [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."spin"] runtime_type = "io.containerd.spin.v2" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."spin".options] BinaryName = "/usr/local/bin/containerd-shim-spin-v2" SystemdCgroup = true [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."wws"] runtime_type = "io.containerd.wws.v1" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."wws".options] BinaryName = "/usr/local/bin/containerd-shim-wws-v1" SystemdCgroup = true ``` $ kgp -A ``` NAMESPACE NAME READY STATUS RESTARTS AGE kube-system coredns-77ccd57875-q2fmm 1/1 Running 0 119s kube-system local-path-provisioner-79ffd768b5-7f7vn 1/1 Running 0 119s kube-system helm-install-traefik-crd-zqnv9 0/1 Completed 0 119s kube-system svclb-traefik-e51d0910-v4gmb 2/2 Running 0 102s kube-system helm-install-traefik-prsh2 0/1 Completed 1 119s kube-system traefik-768bdcdcdd-pgwfd 1/1 Running 0 102s kube-system metrics-server-c44988498-2hpj4 1/1 Running 0 119s ``` $ ls ``` 90-kubelet.conf bin containerd-shim-slight-v1 containerd-shim-wws-v1 containerd-wasm-shims-v1-slight-linux-x86_64.tar.gz containerd-wasm-shims-v2-spin-linux-x86_64.tar.gz install-k3s.sh middleware.yaml spin-shim-deploy.yaml workload.yaml bashrc.d containerd-shim-lunatic-v1 containerd-shim-spin-v2 containerd-wasm-shims-v1-lunatic-linux-x86_64.tar.gz containerd-wasm-shims-v1-wws-linux-x86_64.tar.gz deploy.yaml k3s-config.yaml runtime.yaml tmp-confs ``` $ k apply -f deploy.yaml -f spin-shim-deploy.yaml -f middleware.yaml ``` deployment.apps/wasm-slight created deployment.apps/wasm-spin created middleware.traefik.containo.us/strip-prefix created service/wasm-slight created service/wasm-spin created Warning: annotation "kubernetes.io/ingress.class" is deprecated, please use 'spec.ingressClassName' instead ingress.networking.k8s.io/wasm-ingress created ``` $ export PUBLIC_IP=$(kubectl get ingress wasm-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}') $ curl -v http://$PUBLIC_IP/slight/hello;curl -v http://$PUBLIC_IP/spin/go-hello;curl -v http://$PUBLIC_IP/spin/hello" > " * Trying 1.1.6.233:80... * Connected to 1.1.6.233 (1.1.6.233) port 80 (#0) > GET /slight/hello HTTP/1.1 > Host: 1.1.6.233 > User-Agent: curl/8.0.1 > Accept: */* > < HTTP/1.1 200 OK < Accept: */* < Accept-Encoding: gzip < Access-Control-Allow-Headers: * < Access-Control-Allow-Methods: * < Access-Control-Allow-Origin: * < Access-Control-Expose-Headers: * < Content-Length: 12 < Date: Fri, 08 Mar 2024 21:35:01 GMT < Host: 1.1.6.233 < User-Agent: curl/8.0.1 < X-Forwarded-For: 10.42.0.1 < X-Forwarded-Host: 1.1.6.233 < X-Forwarded-Port: 80 < X-Forwarded-Prefix: /slight < X-Forwarded-Proto: http < X-Forwarded-Server: traefik-768bdcdcdd-pgwfd < X-Real-Ip: 10.42.0.1 < Content-Type: text/plain; charset=utf-8 < * Connection #0 to host 1.1.6.233 left intact hello world!* Trying 1.1.6.233:80... * Connected to 1.1.6.233 (1.1.6.233) port 80 (#0) > GET /spin/go-hello HTTP/1.1 > Host: 1.1.6.233 > User-Agent: curl/8.0.1 > Accept: */* > < HTTP/1.1 200 OK < Content-Length: 728 < Content-Type: text/plain < Date: Fri, 08 Mar 2024 21:35:01 GMT < Foo: bar < == REQUEST == URL: http://1.1.6.233/go-hello Method: GET Headers: "Accept": "*/*" "X-Forwarded-Host": "1.1.6.233" "X-Forwarded-Proto": "http" "X-Forwarded-Server": "traefik-768bdcdcdd-pgwfd" "Spin-Matched-Route": "/go-hello" "Spin-Component-Route": "/go-hello" "Host": "1.1.6.233" "X-Forwarded-Prefix": "/spin" "Spin-Path-Info": "" "Spin-Base-Path": "/" "Spin-Raw-Component-Route": "/go-hello" "Spin-Client-Addr": "10.42.0.8:57014" "User-Agent": "curl/8.0.1" "Accept-Encoding": "gzip" "Spin-Full-Url": "http://1.1.6.233/go-hello" "X-Forwarded-For": "10.42.0.1" "X-Forwarded-Port": "80" "X-Real-Ip": "10.42.0.1" Body: == RESPONSE == Hello Spin Shim! * Connection #0 to host 1.1.6.233 left intact * Closing connection -1 ```