[k8s] exec

[AmitKumarDas]

// - https://erkanerol.github.io/post/how-kubectl-exec-works/
// - https://www.cyberark.com/resources/threat-research-blog/using-kubelet-client-to-attack-the-kubernetes-cluster

[AmitKumarDas]

// When we run “kubectl exec …” in a machine, a process starts
// You can run it in any machine which has an access to k8s api server

// any machine
// create pod
$ kubectl run pody --image=nginx

// any machine
// exec & sleep to make an observation
$ kubectl exec -it pody-xxx -- sh
# sleep 5000

// any machine
// observe kubectl process pid=8507
$ ps -ef |grep kubectl
501  8507  8409   0  7:19PM ttys000    0:00.13 kubectl exec -it pody-xxx -- sh

// any machine
// it has connections to api server (192.168.205.10.6443)
$ netstat -atnv |grep 8507
tcp4       0      0  192.168.205.1.51673    192.168.205.10.6443    ESTABLISHED 131072 131768   8507      0 0x0102 0x00000020
tcp4       0      0  192.168.205.1.51672    192.168.205.10.6443    ESTABLISHED 131072 131768   8507      0 0x0102 0x00000028

[AmitKumarDas]

// - https://github.com/kubernetes/kubernetes/raw/a1f1f0b599e961a5c59b02c349c0ed818b1851a5/staging/src/k8s.io/kubectl/pkg/cmd/exec/exec.go

req := restClient.Post().
  Resource("pods").
  Name(pod.Name).
  Namespace(pod.Namespace).
  SubResource("exec")
req.VersionedParams(&corev1.PodExecOptions{
  Container: containerName,
  Command:   p.Command,
  Stdin:     p.Stdin,
  Stdout:    p.Out != nil,
  Stderr:    p.ErrOut != nil,
  TTY:       t.Raw,
}, scheme.ParameterCodec)

return p.Executor.Execute(
  "POST", 
  req.URL(), 
  p.Config, 
  p.In, 
  p.Out, 
  p.ErrOut, 
  t.Raw, 
  sizeQueue,
)

// in the apiserver logs
POST "/api/v1/namespaces/default/pods/pody-xxx" ... with webservice /api/v1
Connecting to backend proxy (intercepting redirects) 
https://xx.xx.xx.xx:xxxx/exec/default/pody-xxx/nginx?
  command=sh&
  input=1&
  output=1&
  tty=1

Headers: map[Connection:
[Upgrade] Content-Length:[0] 
Upgrade:[SPDY/3.1] User-Agent:
X-Forwarded-For:[xx.xx.xx.xx] X-Stream-Protocol-Version:
[v4.channel.k8s.io v3.channel.k8s.io v2.channel.k8s.io channel.k8s.io]
]

[AmitKumarDas]

// - https://jasonstitt.com/websocket-kubernetes-exec
//
// using websocket client to exec commands in a Pod

// To stream the output of the command we ran with exec,
// we just need a SPDY client. SPDY is not well supported
// in libraries and is basically a deprecated experiment, 
// which Chrome dropped support for in 2016 signaling a 
// move away from SPDY to HTTP/2.

// The Kubernetes API supports more than just SPDY, 
// although it’s what kubectl uses. It doesn’t yet support 
// HTTP/2, but it does support WebSockets.

const fs = require('fs');
const WebSocket = require('ws');

url = 'wss://x.x.x.x:8443/api/v1/namespaces/default/pods/pody/exec?command=echo&command=foo&stderr=true&stdout=true';

sock = new WebSocket(url, {
  ca: fs.readFileSync(`${process.env.HOME}/.minikube/ca.crt`),
  cert: fs.readFileSync(`${process.env.HOME}/.minikube/client.crt`),
  key: fs.readFileSync(`${process.env.HOME}/.minikube/client.key`)
});

sock.on('upgrade', x => console.log('upgrade', x.headers.upgrade));
sock.on('open', () => console.log('open'));
sock.on('message', x => console.log('message', JSON.stringify(x.toString())));
sock.on('close', () => console.log('close'));

// If you’re running the code from a Kubernetes pod, 
// you should use a serviceaccount which will give 
// you a bearer token stored in 
// /var/run/secrets/kubernetes.io/serviceaccount/token 
// instead of a client certificate, and the CA will be at 
// /var/run/secrets/kubernetes.io/serviceaccount/ca.crt

// Protocol
//
// kubectl and oc use the SPDY protocol at the moment, 
// which is being deprecated . The second option is to
// use Websockets, which seems to be the best way. 
// Anyway one of these two protocols, SPDY or WebSockets,
// is required for communication with this endpoint, 
// and the API will refuse requests without Upgrade headers.
//
// HTTP headers
// To provide all the necessary information, the request
// needs to contain the set of headers required by the API.
// Some of them will be handled by your WebSockets client 
// (e.g. Upgrade, etc.), but there are two that need to 
// be provided by the user.
// 
// The first one is Authorization, with a value of Bearer 
// <token> that authenticates the request. For Kubernetes, 
// follow this guide. With OpenShift, simply get the token
// for your user:
//
// oc whoami -t
//
// The other header is Accept, with the value */*. Any other
// value will be rejected with 406 Not Acceptable
//
// Communication protocol
// With all the information in place, the WebSocket should
// be able to establish a connection and the API will start
// communicating. When you write to the WebSocket, the data
// will be passed to standard input (stdin) and on the 
// receiving end of the WebSocket will be standard output 
// (stdout) and error (stderr). The API defines a simple 
// protocol to multiplex stdout and stderr over a single 
// connection. Every message passed through the web socket 
// is prefixed by a single byte that defines which stream 
// the message belongs to.
//
// 0 == stdin  
// 1 == stdout
// 2 == stderr
//
// So for every message received over the socket, you need 
// to get the first byte and decide whether it is stdout or
// stderr. To send data to the API, you need to convert to 
// bytes and prepend 0 to indicate the message belongs in 
// the stdin stream
//
// Connection lifecycle
// One last problem is that there may be proxies and other 
// “roadblocks” on the way to the API, or you may simply 
// reach the TCP timeout. To get around that, send an empty
// message every once in a while to keep the connection busy.

// golang
// remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
// "k8s.io/client-go/tools/remotecommand"

[AmitKumarDas]

// - https://discuss.kubernetes.io/t/go-client-exec-ing-a-shel-command-in-pod/5354

// ExecuteRemoteCommand executes a remote shell command
// on the given pod returns the output from stdout and
// stderr
func ExecuteRemoteCommand(
  pod *v1.Pod, 
  command string,
) (string, string, error) {
  kubeCfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
    clientcmd.NewDefaultClientConfigLoadingRules(),
    &clientcmd.ConfigOverrides{},
  )
  conf, err := kubeCfg.ClientConfig()
  if err != nil {
    return "", "", err
  }
  coreClient, err := corev1client.NewForConfig(conf)
  if err != nil {
    return "", "", err
  }

  buf := &bytes.Buffer{}
  errBuf := &bytes.Buffer{}
  request := coreClient.RESTClient().
    Post().
    Namespace(pod.Namespace).
    Resource("pods").
    Name(pod.Name).
    SubResource("exec").
    VersionedParams(&k8s_corev1.PodExecOptions{
      Command: []string{"/bin/sh", "-c", command},
      Stdin:   true, // this might be false
      Stdout:  true,
      Stderr:  true,
      TTY:     true,
    }, scheme.ParameterCodec)
  exec, err := remotecommand.NewSPDYExecutor(conf, "POST", request.URL())
  err = exec.Stream(remotecommand.StreamOptions{
    Stdout: buf,
    Stderr: errBuf,
  })
  if err != nil {
    return "", "", errors.Wrapf(
      err, 
      "Failed executing command %s on %v/%v", 
      command, 
      pod.Namespace, 
      pod.Name,
    )
  }

  return buf.String(), errBuf.String(), nil
}

[AmitKumarDas]

// - https://github.com/kairen/websocket-exec