Closed AmitKumarDas closed 2 years ago
// When we run “kubectl exec …” in a machine, a process starts
// You can run it in any machine which has an access to k8s api server
// any machine
// create pod
$ kubectl run pody --image=nginx
// any machine
// exec & sleep to make an observation
$ kubectl exec -it pody-xxx -- sh
# sleep 5000
// any machine
// observe kubectl process pid=8507
$ ps -ef |grep kubectl
501 8507 8409 0 7:19PM ttys000 0:00.13 kubectl exec -it pody-xxx -- sh
// any machine
// it has connections to api server (192.168.205.10.6443)
$ netstat -atnv |grep 8507
tcp4 0 0 192.168.205.1.51673 192.168.205.10.6443 ESTABLISHED 131072 131768 8507 0 0x0102 0x00000020
tcp4 0 0 192.168.205.1.51672 192.168.205.10.6443 ESTABLISHED 131072 131768 8507 0 0x0102 0x00000028
// - https://github.com/kubernetes/kubernetes/raw/a1f1f0b599e961a5c59b02c349c0ed818b1851a5/staging/src/k8s.io/kubectl/pkg/cmd/exec/exec.go
req := restClient.Post().
Resource("pods").
Name(pod.Name).
Namespace(pod.Namespace).
SubResource("exec")
req.VersionedParams(&corev1.PodExecOptions{
Container: containerName,
Command: p.Command,
Stdin: p.Stdin,
Stdout: p.Out != nil,
Stderr: p.ErrOut != nil,
TTY: t.Raw,
}, scheme.ParameterCodec)
return p.Executor.Execute(
"POST",
req.URL(),
p.Config,
p.In,
p.Out,
p.ErrOut,
t.Raw,
sizeQueue,
)
// in the apiserver logs
POST "/api/v1/namespaces/default/pods/pody-xxx" ... with webservice /api/v1
Connecting to backend proxy (intercepting redirects)
https://xx.xx.xx.xx:xxxx/exec/default/pody-xxx/nginx?
command=sh&
input=1&
output=1&
tty=1
Headers: map[Connection:
[Upgrade] Content-Length:[0]
Upgrade:[SPDY/3.1] User-Agent:
X-Forwarded-For:[xx.xx.xx.xx] X-Stream-Protocol-Version:
[v4.channel.k8s.io v3.channel.k8s.io v2.channel.k8s.io channel.k8s.io]
]
// - https://jasonstitt.com/websocket-kubernetes-exec
//
// using websocket client to exec commands in a Pod
// To stream the output of the command we ran with exec,
// we just need a SPDY client. SPDY is not well supported
// in libraries and is basically a deprecated experiment,
// which Chrome dropped support for in 2016 signaling a
// move away from SPDY to HTTP/2.
// The Kubernetes API supports more than just SPDY,
// although it’s what kubectl uses. It doesn’t yet support
// HTTP/2, but it does support WebSockets.
const fs = require('fs');
const WebSocket = require('ws');
url = 'wss://x.x.x.x:8443/api/v1/namespaces/default/pods/pody/exec?command=echo&command=foo&stderr=true&stdout=true';
sock = new WebSocket(url, {
ca: fs.readFileSync(`${process.env.HOME}/.minikube/ca.crt`),
cert: fs.readFileSync(`${process.env.HOME}/.minikube/client.crt`),
key: fs.readFileSync(`${process.env.HOME}/.minikube/client.key`)
});
sock.on('upgrade', x => console.log('upgrade', x.headers.upgrade));
sock.on('open', () => console.log('open'));
sock.on('message', x => console.log('message', JSON.stringify(x.toString())));
sock.on('close', () => console.log('close'));
// If you’re running the code from a Kubernetes pod,
// you should use a serviceaccount which will give
// you a bearer token stored in
// /var/run/secrets/kubernetes.io/serviceaccount/token
// instead of a client certificate, and the CA will be at
// /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
// Protocol
//
// kubectl and oc use the SPDY protocol at the moment,
// which is being deprecated . The second option is to
// use Websockets, which seems to be the best way.
// Anyway one of these two protocols, SPDY or WebSockets,
// is required for communication with this endpoint,
// and the API will refuse requests without Upgrade headers.
//
// HTTP headers
// To provide all the necessary information, the request
// needs to contain the set of headers required by the API.
// Some of them will be handled by your WebSockets client
// (e.g. Upgrade, etc.), but there are two that need to
// be provided by the user.
//
// The first one is Authorization, with a value of Bearer
// <token> that authenticates the request. For Kubernetes,
// follow this guide. With OpenShift, simply get the token
// for your user:
//
// oc whoami -t
//
// The other header is Accept, with the value */*. Any other
// value will be rejected with 406 Not Acceptable
//
// Communication protocol
// With all the information in place, the WebSocket should
// be able to establish a connection and the API will start
// communicating. When you write to the WebSocket, the data
// will be passed to standard input (stdin) and on the
// receiving end of the WebSocket will be standard output
// (stdout) and error (stderr). The API defines a simple
// protocol to multiplex stdout and stderr over a single
// connection. Every message passed through the web socket
// is prefixed by a single byte that defines which stream
// the message belongs to.
//
// 0 == stdin
// 1 == stdout
// 2 == stderr
//
// So for every message received over the socket, you need
// to get the first byte and decide whether it is stdout or
// stderr. To send data to the API, you need to convert to
// bytes and prepend 0 to indicate the message belongs in
// the stdin stream
//
// Connection lifecycle
// One last problem is that there may be proxies and other
// “roadblocks” on the way to the API, or you may simply
// reach the TCP timeout. To get around that, send an empty
// message every once in a while to keep the connection busy.
// golang
// remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
// "k8s.io/client-go/tools/remotecommand"
// - https://discuss.kubernetes.io/t/go-client-exec-ing-a-shel-command-in-pod/5354
// ExecuteRemoteCommand executes a remote shell command
// on the given pod returns the output from stdout and
// stderr
func ExecuteRemoteCommand(
pod *v1.Pod,
command string,
) (string, string, error) {
kubeCfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
clientcmd.NewDefaultClientConfigLoadingRules(),
&clientcmd.ConfigOverrides{},
)
conf, err := kubeCfg.ClientConfig()
if err != nil {
return "", "", err
}
coreClient, err := corev1client.NewForConfig(conf)
if err != nil {
return "", "", err
}
buf := &bytes.Buffer{}
errBuf := &bytes.Buffer{}
request := coreClient.RESTClient().
Post().
Namespace(pod.Namespace).
Resource("pods").
Name(pod.Name).
SubResource("exec").
VersionedParams(&k8s_corev1.PodExecOptions{
Command: []string{"/bin/sh", "-c", command},
Stdin: true, // this might be false
Stdout: true,
Stderr: true,
TTY: true,
}, scheme.ParameterCodec)
exec, err := remotecommand.NewSPDYExecutor(conf, "POST", request.URL())
err = exec.Stream(remotecommand.StreamOptions{
Stdout: buf,
Stderr: errBuf,
})
if err != nil {
return "", "", errors.Wrapf(
err,
"Failed executing command %s on %v/%v",
command,
pod.Namespace,
pod.Name,
)
}
return buf.String(), errBuf.String(), nil
}
// - https://github.com/kairen/websocket-exec