Closed zking2000 closed 1 month ago
Using OPA - Gatekeeper
kubectl apply -f https://raw.githubusercontent.com/jcanizalez/gatekeeper/master/deploy/gatekeeper.yaml
apiVersion: templates.gatekeeper.sh/v1
kind: ConstraintTemplate
metadata:
name: k8sdenyexec
spec:
crd:
spec:
names:
kind: k8sdenyexec
targets:
- target: admission.k8s.gatekeeper.sh
rego: |
package k8sdenyexec
import future.keywords.every
violation[{"msg": msg}] {
input.review.object.stdin == true
msg := "Interactive exec is not permitted."
}
allowed_commands := {"ls","cat"}
violation[{"msg": msg}] {
some command in input.review.object.command
not command in allowed_commands
msg := sprintf("Only %v commands are allowed.", [allowed_commands])
}
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: k8sdenyexec
metadata:
name: k8sdenyexec
spec:
match:
kinds:
- apiGroups: [ "" ]
kinds: [ "PodExecOptions" ]
kubectl create deploy nginx --image nginx:latest
deployment.apps/nginx created
root@stephen:/opt/k8s/opa# kubectl get po
NAME READY STATUS RESTARTS AGE
nginx 1/1 Running 0 2m42s
nginx-7bf8c77b5b-ld5r6 0/1 ContainerCreating 0 4s
root@stephen:/opt/k8s/opa# watch kubectl get po
root@stephen:/opt/k8s/opa# kubectl exec -it nginx-7bf8c77b5b-ld5r6 -- bash
Error from server (Forbidden): admission webhook "validation.gatekeeper.sh" denied the request: [k8sdenyexec] Interactive exec is not permitted.
[k8sdenyexec] Only {"cat", "ls"} commands are allowed.
#!/bin/bash
# 检查是否提供了用户名
if [ "$#" -ne 1 ]; then
echo "Usage: $0 <username>"
exit 1
fi
# 设置变量
USERNAME=$1
NAMESPACE="default"
CLUSTER_NAME=$(kubectl config view --minify -o jsonpath='{.clusters[0].name}')
SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}')
# 创建服务账户
kubectl create serviceaccount ${USERNAME} -n ${NAMESPACE}
# 创建角色
cat <<EOF | kubectl apply -f -
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
namespace: ${NAMESPACE}
name: ${USERNAME}-role
rules:
- apiGroups: [""]
resources: ["pods", "services", "configmaps"]
verbs: ["get", "list", "watch"]
EOF
# 创建角色绑定
kubectl create rolebinding ${USERNAME}-rolebinding --role=${USERNAME}-role --serviceaccount=${NAMESPACE}:${USERNAME} -n ${NAMESPACE}
# 创建用于身份验证的token
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Secret
metadata:
name: ${USERNAME}-token
namespace: ${NAMESPACE}
annotations:
kubernetes.io/service-account.name: ${USERNAME}
type: kubernetes.io/service-account-token
EOF
# 获取token
TOKEN=$(kubectl get secret ${USERNAME}-token -n ${NAMESPACE} -o jsonpath='{.data.token}' | base64 --decode)
# 获取集群 CA 证书
CLUSTER_CA=$(kubectl config view --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}')
# 创建 kubeconfig 文件
cat <<EOF > ${USERNAME}-kubeconfig
apiVersion: v1
kind: Config
clusters:
- name: ${CLUSTER_NAME}
cluster:
certificate-authority-data: ${CLUSTER_CA}
server: ${SERVER}
contexts:
- name: ${USERNAME}@${CLUSTER_NAME}
context:
cluster: ${CLUSTER_NAME}
user: ${USERNAME}
namespace: ${NAMESPACE}
current-context: ${USERNAME}@${CLUSTER_NAME}
users:
- name: ${USERNAME}
user:
token: ${TOKEN}
EOF
echo "User ${USERNAME} created successfully."
echo "Kubeconfig file saved as ${USERNAME}-kubeconfig"
echo "Use the following command to test the new user's permissions:"
echo "kubectl --kubeconfig=./${USERNAME}-kubeconfig auth can-i list pods"
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: allen-role
namespace: default
rules:
- apiGroups:
- ""
resources:
- pods
- services
- configmaps
verbs:
- get
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: allen-rolebinding
namespace: default
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: allen-role
subjects:
- kind: ServiceAccount
name: allen
namespace: default
kubectl --kubeconfig=./allen-kubeconfig exec -it nginx -n stephen -- bash
Error from server (Forbidden): pods "nginx" is forbidden: User "system:serviceaccount:default:allen" cannot get resource "pods" in API group "" in the namespace "stephen"
对于 Google Kubernetes Engine (GKE) 集群,禁止 exec 登录确实需要考虑额外的因素,因为 GKE 使用 Google Cloud IAM (Identity and Access Management) 来管理访问控制。以下是在 GKE 环境中禁止 exec 登录的一些考虑和步骤:
Kubernetes RBAC: 首先,你仍然可以使用 Kubernetes RBAC 来限制 exec 权限。例如: yamlCopyapiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: no-exec-role rules:
apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: no-exec-binding subjects:
roles/container.developer roles/container.admin
这些角色默认包含执行 exec 的权限。 自定义 IAM 角色: 你可以创建自定义 IAM 角色,明确不包含 exec 权限。例如: Copygcloud iam roles create noExecKubernetesRole --project=YOUR_PROJECT_ID \ --title="No Exec Kubernetes Role" \ --description="Kubernetes role without exec permissions" \ --permissions=container.clusters.get,container.clusters.list \ --stage=ALPHA 然后将这个角色分配给需要的用户或服务账号。 使用 GKE 的 RBAC 和 Google Groups 集成: GKE 允许你将 Google Groups 与 Kubernetes RBAC 集成。你可以创建一个 Google Group,将用户添加到该组,然后在 Kubernetes RBAC 中引用这个组。 网络策略: 虽然不是直接禁止 exec,但你可以使用网络策略来限制 Pod 的出站连接,这在某种程度上可以限制 exec 的有效性。 审计日志: 启用 GKE 审计日志,以监控和记录所有 exec 操作。 OPA Gatekeeper 或 Admission Controllers: 考虑使用 Open Policy Agent (OPA) Gatekeeper 或自定义的 Admission Controller 来强制执行更细粒度的策略,包括禁止 exec。
实施步骤:
审查并修改现有的 IAM 角色分配,确保用户没有不必要的 exec 权限。 实施 Kubernetes RBAC 策略来限制 exec。 创建并分配自定义 IAM 角色,不包含 exec 权限。 配置 GKE 集群以使用 Google Groups 进行 RBAC。 实施网络策略来限制 Pod 的出站连接。 启用审计日志以监控 exec 操作。 考虑实施 OPA Gatekeeper 或自定义 Admission Controller。
请注意,完全禁止 exec 可能会影响某些合法的运维操作。确保在实施这些限制之前,有其他方法可以进行必要的调试和管理任务。