zking2000 / NotePad

1 stars 0 forks source link

webhook #78

Closed zking2000 closed 1 month ago

zking2000 commented 1 month ago

from flask import Flask, request, jsonify
import json
from base64 import b64decode
import jsonpatch

app = Flask(__name__)

def create_patch(pod):
    sidecar_config = {
        "name": "otel-collector",
        "image": "otel/opentelemetry-collector:latest",
        "ports": [
            {"containerPort": 4317},
            {"containerPort": 4318}
        ],
        "volumeMounts": [
            {
                "name": "otel-collector-config",
                "mountPath": "/conf"
            }
        ],
        "args": ["--config=/conf/otel-collector-config.yaml"]
    }

    patch = jsonpatch.JsonPatch([
        {"op": "add", "path": "/spec/containers/-", "value": sidecar_config}
    ])

    return patch

@app.route('/mutate', methods=['POST'])
def mutate():
    admission_review = request.json

    try:
        # Extract the Pod object from the AdmissionReview
        pod = json.loads(b64decode(admission_review['request']['object']['raw']))

        # Create the patch
        patch = create_patch(pod)

        # Prepare the AdmissionResponse
        admission_response = {
            "allowed": True,
            "uid": admission_review['request']['uid'],
            "patch": b64encode(str(patch).encode()).decode(),
            "patchType": "JSONPatch"
        }

        # Prepare the AdmissionReview response
        admission_review_response = {
            "apiVersion": "admission.k8s.io/v1",
            "kind": "AdmissionReview",
            "response": admission_response
        }

        return jsonify(admission_review_response)

    except Exception as e:
        print(f"Error processing request: {str(e)}")
        return jsonify({"allowed": False, "status": {"message": str(e)}})

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8443, ssl_context=('/etc/webhook/certs/tls.crt', '/etc/webhook/certs/tls.key'))

FROM python:3.9-slim

WORKDIR /app

COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt

COPY webhook.py .

CMD ["python", "webhook.py"]

Flask==2.0.1
jsonpatch==1.32

apiVersion: v1
kind: ConfigMap
metadata:
  name: otel-collector-config
  namespace: your-namespace
data:
  otel-collector-config.yaml: |
    receivers:
      otlp:
        protocols:
          grpc:
          http:
    exporters:
      # Configure your desired exporter here
      logging:
        loglevel: debug
    processors:
      batch:
    service:
      pipelines:
        traces:
          receivers: [otlp]
          processors: [batch]
          exporters: [logging]

apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: otel-sidecar-injector
webhooks:
- name: otel-sidecar-injector.your-domain.com
  clientConfig:
    service:
      name: otel-sidecar-injector
      namespace: your-namespace
      path: "/mutate"
    caBundle: <base64-encoded-ca-cert>
  rules:
  - apiGroups: [""]
    apiVersions: ["v1"]
    operations: ["CREATE"]
    resources: ["pods"]
    scope: "Namespaced"
  namespaceSelector:
    matchLabels:
      otel-injection: enabled
  admissionReviewVersions: ["v1", "v1beta1"]
  sideEffects: None
  timeoutSeconds: 5

apiVersion: apps/v1
kind: Deployment
metadata:
  name: otel-sidecar-injector
  namespace: your-namespace
  labels:
    app: otel-sidecar-injector
spec:
  replicas: 1
  selector:
    matchLabels:
      app: otel-sidecar-injector
  template:
    metadata:
      labels:
        app: otel-sidecar-injector
    spec:
      containers:
      - name: otel-sidecar-injector
        image: your-registry/otel-sidecar-injector:latest
        imagePullPolicy: Always
        ports:
        - containerPort: 8443
        volumeMounts:
        - name: webhook-certs
          mountPath: /etc/webhook/certs
          readOnly: true
      volumes:
      - name: webhook-certs
        secret:
          secretName: webhook-certs
---
apiVersion: v1
kind: Service
metadata:
  name: otel-sidecar-injector
  namespace: your-namespace
spec:
  selector:
    app: otel-sidecar-injector
  ports:
  - port: 443
    targetPort: 8443

#!/bin/bash

set -e

usage() {
    cat <<EOF
Generate certificate suitable for use with an sidecar-injector webhook service.

This script uses k8s' CertificateSigningRequest API to a generate a
certificate signed by k8s CA suitable for use with sidecar-injector webhook
services. This requires permissions to create and approve CSR. See
https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster for
detailed explantion and additional instructions.

The server key/cert k8s CA cert are stored in a k8s secret.

usage: ${0} [OPTIONS]

The following flags are required.

    --service          Service name of webhook.
    --namespace        Namespace where webhook service and secret reside.
    --secret           Secret name for CA certificate and server certificate/key pair.

The following flags are optional.

    --webhook-name     Webhook config name, e.g. "otel-sidecar-injector.your-domain.com".
                       If not provided, the webhook name is computed from the service name and namespace.

EOF
    exit 1
}

while [[ $# -gt 0 ]]; do
    case ${1} in
        --service)
            service="$2"
            shift
            ;;
        --webhook-name)
            webhook_name="$2"
            shift
            ;;
        --namespace)
            namespace="$2"
            shift
            ;;
        --secret)
            secret="$2"
            shift
            ;;
        *)
            usage
            ;;
    esac
    shift
done

[ -z ${service} ] && service=otel-sidecar-injector
[ -z ${webhook_name} ] && webhook_name=${service}.${namespace}.svc
[ -z ${secret} ] && secret=webhook-certs
[ -z ${namespace} ] && namespace=default

if [ ! -x "$(command -v openssl)" ]; then
    echo "openssl not found"
    exit 1
fi

csrName=${service}.${namespace}
tmpdir=$(mktemp -d)
echo "creating certs in tmpdir ${tmpdir} "

cat <<EOF >> ${tmpdir}/csr.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = ${service}
DNS.2 = ${service}.${namespace}
DNS.3 = ${service}.${namespace}.svc
EOF

openssl genrsa -out ${tmpdir}/server-key.pem 2048
openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf

# clean-up any previously created CSR for our service. Ignore errors if not present.
kubectl delete csr ${csrName} 2>/dev/null || true

# create  server cert/key CSR and  send to k8s API
cat <<EOF | kubectl create -f -
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: ${csrName}
spec:
  groups:
  - system:authenticated
  request: $(cat ${tmpdir}/server.csr | base64 | tr -d '\n')
  signerName: kubernetes.io/kubelet-serving
  usages:
  - digital signature
  - key encipherment
  - server auth
EOF

# verify CSR has been created
while true; do
    kubectl get csr ${csrName}
    if [ "$?" -eq 0 ]; then
        break
    fi
done

# approve and fetch the signed certificate
kubectl certificate approve ${csrName}
# verify certificate has been signed
for x in $(seq 10); do
    serverCert=$(kubectl get csr ${csrName} -o jsonpath='{.status.certificate}')
    if [[ ${serverCert} != '' ]]; then
        break
    fi
    sleep 1
done
if [[ ${serverCert} == '' ]]; then
    echo "ERROR: After approving csr ${csrName}, the signed certificate did not appear on the resource.

zking2000 commented 1 month ago

#!/bin/bash

set -e

usage() {
    cat <<EOF
Generate certificate suitable for use with an sidecar-injector webhook service.

This script uses k8s' CertificateSigningRequest API to generate a
certificate signed by k8s CA suitable for use with sidecar-injector webhook
services. This requires permissions to create and approve CSR. See
https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster for
detailed explanation and additional instructions.

The server key/cert k8s CA cert are stored in a k8s secret.

usage: ${0} [OPTIONS]

The following flags are required.

    --service          Service name of webhook.
    --namespace        Namespace where webhook service and secret reside.
    --secret           Secret name for CA certificate and server certificate/key pair.

The following flags are optional.

    --webhook-name     Webhook config name, e.g. "otel-sidecar-injector.your-domain.com".
                       If not provided, the webhook name is computed from the service name and namespace.

EOF
    exit 1
}

while [[ $# -gt 0 ]]; do
    case ${1} in
        --service)
            service="$2"
            shift
            ;;
        --webhook-name)
            webhook_name="$2"
            shift
            ;;
        --namespace)
            namespace="$2"
            shift
            ;;
        --secret)
            secret="$2"
            shift
            ;;
        *)
            usage
            ;;
    esac
    shift
done

[ -z ${service} ] && service=otel-sidecar-injector
[ -z ${webhook_name} ] && webhook_name=${service}.${namespace}.svc
[ -z ${secret} ] && secret=webhook-certs
[ -z ${namespace} ] && namespace=default

if [ ! -x "$(command -v openssl)" ]; then
    echo "openssl not found"
    exit 1
fi

csrName=${service}.${namespace}
tmpdir=$(mktemp -d)
echo "creating certs in tmpdir ${tmpdir} "

cat <<EOF >> ${tmpdir}/csr.conf
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names
[alt_names]
DNS.1 = ${service}
DNS.2 = ${service}.${namespace}
DNS.3 = ${service}.${namespace}.svc
EOF

openssl genrsa -out ${tmpdir}/server-key.pem 2048
openssl req -new -key ${tmpdir}/server-key.pem -subj "/CN=${service}.${namespace}.svc" -out ${tmpdir}/server.csr -config ${tmpdir}/csr.conf

# Clean-up any previously created CSR for our service. Ignore errors if not present.
kubectl delete csr ${csrName} 2>/dev/null || true

# Create server cert/key CSR and send to k8s API
cat <<EOF | kubectl create -f -
apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: ${csrName}
spec:
  request: $(cat ${tmpdir}/server.csr | base64 | tr -d '\n')
  signerName: kubernetes.io/kubelet-serving
  usages:
  - digital signature
  - key encipherment
  - server auth
EOF

# Verify CSR has been created
while true; do
    kubectl get csr ${csrName}
    if [ "$?" -eq 0 ]; then
        break
    fi
done

# Approve and fetch the signed certificate
kubectl certificate approve ${csrName}

# Verify certificate has been signed
for x in $(seq 10); do
    serverCert=$(kubectl get csr ${csrName} -o jsonpath='{.status.certificate}')
    if [[ ${serverCert} != '' ]]; then
        break
    fi
    sleep 1
done
if [[ ${serverCert} == '' ]]; then
    echo "ERROR: After approving csr ${csrName}, the signed certificate did not appear on the resource. Giving up after 10 attempts." >&2
    exit 1
fi
echo ${serverCert} | openssl base64 -d -A -out ${tmpdir}/server-cert.pem

# Create the secret with CA cert and server cert/key
kubectl create secret generic ${secret} \
        --from-file=key.pem=${tmpdir}/server-key.pem \
        --from-file=cert.pem=${tmpdir}/server-cert.pem \
        --dry-run=client -o yaml |
    kubectl -n ${namespace} apply -f -

# Encode the CA
caBundle=$(kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n')

cat <<EOF | kubectl create -f -
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
  name: ${webhook_name}
webhooks:
- name: ${webhook_name}
  clientConfig:
    service:
      name: ${service}
      namespace: ${namespace}
      path: "/mutate"
    caBundle: ${caBundle}
  rules:
  - operations: ["CREATE"]
    apiGroups: [""]
    apiVersions: ["v1"]
    resources: ["pods"]
  admissionReviewVersions: ["v1", "v1beta1"]
  sideEffects: None
  timeoutSeconds: 5
EOF

# Clean-up
rm -rf ${tmpdir}

echo "The webhook has been deployed and configured."

zking2000 commented 1 month ago

理解您的需求。为了找出webhook调用失败的原因，我们确实需要添加更多的日志。以下是我建议在代码中添加的日志：

在mutate()函数的开始处添加日志：

def mutate():
    print("Entering mutate function")
    admission_review = request.json
    print(f"Received admission review: {admission_review}")

在try块中的关键步骤添加日志：

try:
    print("Extracting Pod object")
    pod = json.loads(b64decode(admission_review['request']['object']['raw']))
    print(f"Extracted pod: {pod}")

    print("Creating patch")
    patch = create_patch(pod)
    print(f"Created patch: {patch}")

    print("Preparing AdmissionResponse")
    admission_response = {
        "allowed": True,
        "uid": admission_review['request']['uid'],
        "patch": b64encode(str(patch).encode()).decode(),
        "patchType": "JSONPatch"
    }
    print(f"Prepared admission response: {admission_response}")

    print("Preparing AdmissionReview response")
    admission_review_response = {
        "apiVersion": "admission.k8s.io/v1",
        "kind": "AdmissionReview",
        "response": admission_response
    }
    print(f"Prepared admission review response: {admission_review_response}")

    return jsonify(admission_review_response)

在create_patch函数中添加日志：

def create_patch(pod):
    print(f"Creating patch for pod: {pod}")
    sidecar_config = {
        "volumeMounts": [
            {
                "mountPath": "/etc/otelcol"
            }
        ],
        "args": ["--config=/etc/otelcol/config.yaml"]
    }
    print(f"Sidecar config: {sidecar_config}")

    patch = jsonpatch.JsonPatch([
        {"op": "add", "path": "/spec/containers/-", "value": sidecar_config}
    ])
    print(f"Created patch: {patch}")
    return patch

在主函数中添加启动日志：

if __name__ == '__main__':
    print("Starting webhook server")
    app.run(host='0.0.0.0', port=9443, ssl_context=('/etc/webhook/certs/cert.pem', '/etc/webhook/certs/key.pem'))

这些日志将帮助您追踪webhook的执行流程，并在出现问题时提供更多上下文信息。特别要注意的是：

检查接收到的admission review的内容是否正确。
确保Pod对象能够正确解析。
验证patch的创建是否成功。
确认响应的格式是否符合Kubernetes的要求。

添加这些日志后，您应该能够看到webhook处理请求的每个步骤，这将有助于定位失败的具体位置和原因。如果问题仍然存在，可能需要进一步检查：

SSL证书配置是否正确。
网络连接是否存在问题。
Kubernetes API服务器是否能够正确访问webhook服务。

如果您需要进一步的帮助或有任何问题，请随时告诉我。

zking2000 commented 1 month ago

当然，我可以为您提供一个用于测试webhook的curl命令。这个命令将模拟Kubernetes API服务器向您的webhook发送的请求。以下是一个示例curl命令：

curl -k -v -X POST https://localhost:9443/mutate \
  -H "Content-Type: application/json" \
  -d '{
    "kind": "AdmissionReview",
    "apiVersion": "admission.k8s.io/v1",
    "request": {
      "uid": "e911857d-c318-4f58-b8e9-6191e6cc1c89",
      "kind": {"group":"", "version":"v1", "kind":"Pod"},
      "resource": {"group":"", "version":"v1", "resource":"pods"},
      "operation": "CREATE",
      "object": {
        "metadata": {"name": "test-pod", "namespace": "default"},
        "spec": {
          "containers": [
            {
              "name": "test-container",
              "image": "nginx:latest"
            }
          ]
        }
      },
      "oldObject": null,
      "dryRun": false
    }
  }'

这个命令的解释：

-k: 允许不安全的服务器连接（如果您使用的是自签名证书）
-v: 详细模式，显示更多的调试信息
-X POST: 指定HTTP方法为POST
https://localhost:9443/mutate: webhook的URL（根据您的实际配置调整）
-H "Content-Type: application/json": 设置内容类型为JSON
-d '...': 请求体，包含一个完整的AdmissionReview对象

请注意以下几点：

确保将URL（https://localhost:9443/mutate）替换为您的webhook实际地址。
如果您的webhook需要认证，可能需要添加适当的认证头。
根据您的实际需求，您可能需要调整请求体中的Pod规范。
如果您使用的是自签名证书，-k选项将跳过证书验证。在生产环境中，应该使用有效的证书并移除此选项。
运行此命令时，确保您的webhook服务器正在运行。

执行此命令后，您应该能看到webhook的响应，包括它是否允许请求以及任何修改（patch）。这将帮助您验证webhook是否按预期工作。

如果遇到任何问题或需要进一步的解释，请随时告诉我。

zking2000 commented 1 month ago

当然，我会为您提供修改过的代码。这个版本增加了更详细的错误处理和日志记录，应该能帮助您更好地诊断问题：

import json
from base64 import b64decode, b64encode
from flask import Flask, request, jsonify
import jsonpatch

app = Flask(__name__)

def create_patch(pod):
    print(f"Creating patch for pod: {pod}")
    sidecar_config = {
        "name": "sidecar-container",
        "image": "sidecar-image:latest",
        "volumeMounts": [
            {
                "name": "config-volume",
                "mountPath": "/etc/otelcol"
            }
        ],
        "args": ["--config=/etc/otelcol/config.yaml"]
    }
    patch = jsonpatch.JsonPatch([
        {"op": "add", "path": "/spec/containers/-", "value": sidecar_config}
    ])
    print(f"Created patch: {patch}")
    return patch

@app.route('/mutate', methods=['POST'])
def mutate():
    print("Entering mutate function")
    try:
        admission_review = request.json
        print(f"Received admission review: {json.dumps(admission_review, indent=2)}")

        if not admission_review or 'request' not in admission_review:
            raise KeyError("Invalid AdmissionReview structure")

        print("Extracting Pod object")
        raw_data = admission_review['request']['object']['raw']
        print(f"Raw data: {raw_data}")

        decoded_data = b64decode(raw_data).decode('utf-8')
        print(f"Decoded data: {decoded_data}")

        pod = json.loads(decoded_data)
        print(f"Extracted pod: {json.dumps(pod, indent=2)}")

        print("Creating patch")
        patch = create_patch(pod)
        print(f"Created patch: {patch}")

        print("Preparing AdmissionResponse")
        admission_response = {
            "allowed": True,
            "uid": admission_review['request']['uid'],
            "patch": b64encode(str(patch).encode()).decode(),
            "patchType": "JSONPatch"
        }
        print(f"Prepared admission response: {json.dumps(admission_response, indent=2)}")

        print("Preparing AdmissionReview response")
        admission_review_response = {
            "apiVersion": "admission.k8s.io/v1",
            "kind": "AdmissionReview",
            "response": admission_response
        }
        print(f"Prepared admission review response: {json.dumps(admission_review_response, indent=2)}")

        return jsonify(admission_review_response)

    except KeyError as e:
        error_msg = f"KeyError: Missing key in request: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})
    except json.JSONDecodeError as e:
        error_msg = f"JSONDecodeError: Invalid JSON in request: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})
    except Exception as e:
        error_msg = f"Unexpected error: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})

if __name__ == '__main__':
    print("Starting webhook server")
    app.run(host='0.0.0.0', port=9443, ssl_context=('/etc/webhook/certs/cert.pem', '/etc/webhook/certs/key.pem'))

这个修改后的版本包含以下改进：

更详细的错误处理，区分了不同类型的异常（KeyError, JSONDecodeError, 和其他未预期的异常）。
增加了更多的日志输出，包括原始数据、解码后的数据等中间步骤的结果。
使用 json.dumps() 带缩进来打印复杂的对象，使日志更易读。
在处理 admission_review 之前增加了额外的检查。
解码 base64 数据时，显式地将结果转换为 UTF-8 字符串。

使用这个版本的代码，您应该能够获得更详细的错误信息和日志输出。这将有助于识别问题的根源，无论是数据格式问题、解码问题还是其他未预料到的问题。

在运行这个修改后的webhook时，请密切关注控制台输出或日志文件。如果仍然遇到问题，请提供完整的日志输出，这将帮助进一步诊断问题。

另外，请确保您的 SSL 证书配置正确，或在测试环境中使用 -k 选项跳过 SSL 验证。

zking2000 commented 1 month ago

根据提供的图片，我们可以看到webhook仍然返回了一个错误，特别是 {"allowed":false,"status":{"message":"'raw'"}} 这个响应。这表明问题仍然存在于处理 'raw' 字段的部分。让我们进一步分析并提供解决方案：

请求格式问题：从图片顶部可以看到，AdmissionReview 对象中没有 'raw' 字段。相反，整个 Pod 规范直接作为 'object' 字段的值。
代码修改建议：我们需要修改代码以适应这种请求格式。以下是建议的修改：

@app.route('/mutate', methods=['POST'])
def mutate():
    print("Entering mutate function")
    try:
        admission_review = request.json
        print(f"Received admission review: {json.dumps(admission_review, indent=2)}")

        if not admission_review or 'request' not in admission_review:
            raise KeyError("Invalid AdmissionReview structure")

        print("Extracting Pod object")
        pod = admission_review['request']['object']
        print(f"Extracted pod: {json.dumps(pod, indent=2)}")

        print("Creating patch")
        patch = create_patch(pod)
        print(f"Created patch: {patch}")

        print("Preparing AdmissionResponse")
        admission_response = {
            "allowed": True,
            "uid": admission_review['request']['uid'],
            "patch": b64encode(str(patch).encode()).decode(),
            "patchType": "JSONPatch"
        }
        print(f"Prepared admission response: {json.dumps(admission_response, indent=2)}")

        print("Preparing AdmissionReview response")
        admission_review_response = {
            "apiVersion": "admission.k8s.io/v1",
            "kind": "AdmissionReview",
            "response": admission_response
        }
        print(f"Prepared admission review response: {json.dumps(admission_review_response, indent=2)}")

        return jsonify(admission_review_response)

    except KeyError as e:
        error_msg = f"KeyError: Missing key in request: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})
    except json.JSONDecodeError as e:
        error_msg = f"JSONDecodeError: Invalid JSON in request: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})
    except Exception as e:
        error_msg = f"Unexpected error: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})

主要变化：

移除了 'raw' 字段的处理逻辑
直接使用 admission_review['request']['object'] 作为 Pod 对象

SSL证书问题：虽然不是导致当前错误的直接原因，但图片中显示SSL证书验证失败。在生产环境中，你应该解决这个问题。对于测试，可以使用 -k 选项跳过SSL验证。

测试步骤：

使用修改后的代码更新你的webhook服务器。
重启webhook服务器。

使用以下命令重新测试（注意 -k 选项用于跳过SSL验证）：

curl -k -v -X POST https://otel-sidecar-injector.auto-sidecar-injector.svc.cluster.local/mutate \
-H "Content-Type: application/json" \
-d '{"kind":"AdmissionReview","apiVersion":"admission.k8s.io/v1","request":{"uid":"e911857d-c318-4f58-b8e9-6191e6cc1c89","kind":{"group":"","version":"v1","kind":"Pod"},"resource":{"group":"","version":"v1","resource":"pods"},"operation":"CREATE","object":{"metadata":{"name":"test-pod","namespace":"default"},"spec":{"containers":[{"name":"test-container","image":"nginx:latest"}]}},"oldObject":null,"dryRun":false}}'

这些修改应该能解决你遇到的 'raw' 字段问题。如果你仍然遇到问题，请提供更新后的错误消息或日志输出，我会很乐意进一步协助你。

zking2000 commented 1 month ago

根据新的错误消息，问题出在 b64encode 函数的使用上。错误提示说 "Unexpected error: name 'b64encode' is not defined"。这意味着 b64encode 函数未被正确导入或定义。让我们修复这个问题：

首先，确保在文件顶部正确导入了 b64encode 函数：

from base64 import b64encode

如果你已经导入了这个函数但仍然遇到这个错误，可能是因为在某处错误地使用了 b64encode。让我们修改 mutate 函数中使用 b64encode 的部分：

import json
from base64 import b64encode
from flask import Flask, request, jsonify
import jsonpatch

app = Flask(__name__)

def create_patch(pod):
    print(f"Creating patch for pod: {pod}")
    sidecar_config = {
        "name": "sidecar-container",
        "image": "sidecar-image:latest",
        "volumeMounts": [
            {
                "name": "config-volume",
                "mountPath": "/etc/otelcol"
            }
        ],
        "args": ["--config=/etc/otelcol/config.yaml"]
    }
    patch = jsonpatch.JsonPatch([
        {"op": "add", "path": "/spec/containers/-", "value": sidecar_config}
    ])
    print(f"Created patch: {patch}")
    return patch

@app.route('/mutate', methods=['POST'])
def mutate():
    print("Entering mutate function")
    try:
        admission_review = request.json
        print(f"Received admission review: {json.dumps(admission_review, indent=2)}")

        if not admission_review or 'request' not in admission_review:
            raise KeyError("Invalid AdmissionReview structure")

        print("Extracting Pod object")
        pod = admission_review['request']['object']
        print(f"Extracted pod: {json.dumps(pod, indent=2)}")

        print("Creating patch")
        patch = create_patch(pod)
        print(f"Created patch: {patch}")

        print("Preparing AdmissionResponse")
        admission_response = {
            "allowed": True,
            "uid": admission_review['request']['uid'],
            "patch": b64encode(str(patch).encode()).decode(),
            "patchType": "JSONPatch"
        }
        print(f"Prepared admission response: {json.dumps(admission_response, indent=2)}")

        print("Preparing AdmissionReview response")
        admission_review_response = {
            "apiVersion": "admission.k8s.io/v1",
            "kind": "AdmissionReview",
            "response": admission_response
        }
        print(f"Prepared admission review response: {json.dumps(admission_review_response, indent=2)}")

        return jsonify(admission_review_response)

    except KeyError as e:
        error_msg = f"KeyError: Missing key in request: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})
    except json.JSONDecodeError as e:
        error_msg = f"JSONDecodeError: Invalid JSON in request: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})
    except Exception as e:
        error_msg = f"Unexpected error: {str(e)}"
        print(error_msg)
        return jsonify({"allowed": False, "status": {"message": error_msg}})

if __name__ == '__main__':
    print("Starting webhook server")
    app.run(host='0.0.0.0', port=9443, ssl_context=('/etc/webhook/certs/cert.pem', '/etc/webhook/certs/key.pem'))

这个修改后的版本应该解决了 b64encode 未定义的问题。主要的变化是确保在文件顶部正确导入了 b64encode 函数。

请使用这个更新后的代码，然后重新启动你的webhook服务器。之后，再次使用之前提供的curl命令来测试webhook。

如果你仍然遇到问题，请提供新的错误信息或日志输出。这将有助于进一步诊断和解决问题。

zking2000 commented 1 month ago

根据您之前的 curl 命令和现在遇到的问题，我建议您修改 curl 命令以确保包含正确的 namespace 信息。以下是修改后的 curl 命令：

curl -k -X POST https://otel-sidecar-injector.admission-control.svc.cluster.local:443/mutate-pods \
-H "Content-Type: application/json" \
-d '{
  "kind": "AdmissionReview",
  "apiVersion": "admission.k8s.io/v1",
  "request": {
    "uid": "e911857d-c318-4f58-b8e9-6191e6cc1e89",
    "kind": {"group":"","version":"v1","kind":"Pod"},
    "resource": {"group":"","version":"v1","resource":"pods"},
    "requestKind": {"group":"","version":"v1","kind":"Pod"},
    "requestResource": {"group":"","version":"v1","resource":"pods"},
    "name": "test-pod",
    "namespace": "default",
    "operation": "CREATE",
    "object": {
      "metadata": {
        "name": "test-pod",
        "namespace": "default"
      },
      "spec": {
        "containers": [
          {
            "name": "test-container",
            "image": "nginx:latest"
          }
        ]
      }
    },
    "oldObject": null,
    "dryRun": true
  }
}'

这个修改后的 curl 命令主要做了以下更改：

确保在 request 对象中包含 "namespace": "default"。
在 object.metadata 中也包含 "namespace": "default"。
保留了原有的 Pod 和容器信息。
使用 "dryRun": true 来安全地测试 webhook，不会实际创建 Pod。

使用这个修改后的 curl 命令，您的 webhook 应该能够正确接收到 namespace 信息，从而避免之前的 "namespace is required" 错误。

如果您使用的是不同的 namespace，请相应地将 "default" 替换为您实际使用的 namespace 名称。

另外，请确保您的 webhook 服务器正在监听正确的路径（在这个例子中是 /mutate-pods）。如果您的路径不同，请相应地调整 curl 命令中的 URL。

在运行此命令之前，请确保您已经正确配置了与集群的连接，并且 webhook 服务已经正确部署和配置。

zking2000 commented 1 month ago

要使用 GKE (Google Kubernetes Engine) 的 CA 签发证书，您可以利用 Kubernetes 的证书签名请求（CSR）功能。以下是具体步骤：

创建私钥：
```
openssl genrsa -out server.key 2048
```

创建证书签名请求（CSR）：

openssl req -new -key server.key -out server.csr -subj "/CN=otelcol-injector.admission-control.svc"

确保 CN (Common Name) 与您的服务名称匹配。

将 CSR 编码为 base64：
```
cat server.csr | base64 | tr -d '\n'
```

创建 Kubernetes CSR 资源（保存为 csr.yaml）：

apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
 name: otelcol-injector-csr
spec:
 request: <base64-encoded-csr>
 signerName: kubernetes.io/kubelet-serving
 usages:
 - digital signature
 - key encipherment
 - server auth

将替换为上一步得到的 base64 编码 CSR。

提交 CSR 到 Kubernetes：
```
kubectl apply -f csr.yaml
```

批准 CSR：

kubectl certificate approve otelcol-injector-csr

获取签名后的证书：

kubectl get csr otelcol-injector-csr -o jsonpath='{.status.certificate}' | base64 --decode > server.crt

创建包含证书和私钥的 Secret：

kubectl create secret tls otelcol-injector-tls --cert=server.crt --key=server.key -n admission-control

在您的 Webhook 部署中使用这个 Secret：

apiVersion: apps/v1
kind: Deployment
metadata:
 name: otelcol-injector
 namespace: admission-control
spec:
 template:
   spec:
     containers:
     - name: webhook
       volumeMounts:
       - name: tls
         mountPath: /etc/webhook/certs
         readOnly: true
     volumes:
     - name: tls
       secret:
         secretName: otelcol-injector-tls

更新 MutatingWebhookConfiguration，使用正确的 caBundle：
```
kubectl get configmap -n kube-system extension-apiserver-authentication -o jsonpath='{.data.client-ca-file}' | base64 | tr -d '\n'
```
将输出的 base64 编码 CA 证书复制到您的 MutatingWebhookConfiguration 的 caBundle 字段中。

这些步骤将使用 GKE 的 CA 为您的 webhook 服务创建和签署证书。确保更新您的 webhook 服务以使用新的证书，并重新部署。这应该能解决 TLS 相关的问题，并有助于解决您之前遇到的超时错误。

zking2000 commented 1 month ago

非常理解您希望保持在谷歌云生态系统内的意愿。对于 GKE 环境，我们可以利用 GKE 提供的内置功能来处理证书问题，而无需引入额外的证书管理系统。以下是一种更适合 GKE 的方法：

使用 GKE 的 Workload Identity： Workload Identity 允许您的 Kubernetes 服务账户使用 Google Cloud IAM 服务账户的身份。这可以简化证书管理过程。

a. 启用 Workload Identity（如果尚未启用）：

  gcloud container clusters update [CLUSTER_NAME] --workload-pool=[PROJECT_ID].svc.id.goog

b. 创建一个 Kubernetes 服务账户：

  kubectl create serviceaccount otelcol-injector-sa -n admission-control

c. 创建一个 Google Cloud IAM 服务账户：

  gcloud iam service-accounts create otelcol-injector-gsa

d. 绑定 Kubernetes 服务账户和 Google Cloud 服务账户：

  gcloud iam service-accounts add-iam-policy-binding otelcol-injector-gsa@[PROJECT_ID].iam.gserviceaccount.com \
    --role roles/iam.workloadIdentityUser \
    --member "serviceAccount:[PROJECT_ID].svc.id.goog[admission-control/otelcol-injector-sa]"

使用 GKE 托管证书： GKE 可以自动管理和更新您的证书。

a. 为您的服务创建一个托管证书：

  apiVersion: networking.gke.io/v1
  kind: ManagedCertificate
  metadata:
    name: otelcol-injector-cert
    namespace: admission-control
  spec:
    domains:
      - otelcol-injector.admission-control.svc.cluster.local

b. 在您的服务中使用这个证书：

  apiVersion: v1
  kind: Service
  metadata:
    name: otelcol-injector
    namespace: admission-control
    annotations:
      networking.gke.io/managed-certificates: otelcol-injector-cert
  spec:
    # ... 其他服务配置 ...

更新 Webhook 配置：在 MutatingWebhookConfiguration 中，使用 GKE 的内置 CA：

apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
 name: otelcol-injector
webhooks:
- name: otelcol-injector.admission-control.svc.cluster.local
 clientConfig:
   caBundle: $(kubectl get configmap -n kube-system kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 | tr -d '\n')
 # ... 其他配置 ...

部署您的 Webhook：在部署中使用 Workload Identity：

apiVersion: apps/v1
kind: Deployment
metadata:
 name: otelcol-injector
 namespace: admission-control
spec:
 template:
   metadata:
     annotations:
       iam.gke.io/gcp-service-account: otelcol-injector-gsa@[PROJECT_ID].iam.gserviceaccount.com
   spec:
     serviceAccountName: otelcol-injector-sa
     containers:
     - name: webhook
       # ... 其他容器配置 ...

这种方法利用了 GKE 的内置功能，无需引入额外的证书管理系统。它应该能解决证书验证问题，同时保持在谷歌云生态系统内。

如果在实施过程中遇到任何问题，请随时告诉我，我会很乐意提供进一步的帮助。

zking2000 commented 1 month ago

如果您使用了 GKE 的托管证书，您的代码需要做一些调整。GKE 托管证书主要用于负载均衡器和入口资源，而不是直接用于 Pod 中的应用程序。对于您的 webhook 服务，我们需要使用 Kubernetes 秘密来存储证书。以下是如何修改您的代码：

首先，创建一个 Kubernetes Secret 来存储证书和密钥：

kubectl create secret tls otelcol-injector-tls \
 --cert=/path/to/tls.crt \
 --key=/path/to/tls.key \
 -n admission-control

在您的 Deployment 中，挂载这个 Secret：

spec:
 template:
   spec:
     containers:
     - name: webhook
       volumeMounts:
       - name: tls
         mountPath: /tls
         readOnly: true
     volumes:
     - name: tls
       secret:
         secretName: otelcol-injector-tls

然后，修改您的代码如下：

if __name__ == "__main__":
   app.run(host='0.0.0.0', port=9443, ssl_context=('/tls/tls.crt', '/tls/tls.key'))

这样，您的应用程序将使用挂载的证书和密钥，而不是直接引用文件路径。

确保您的 MutatingWebhookConfiguration 使用正确的 caBundle：

webhooks:
- name: otelcol-injector.admission-control.svc
 clientConfig:
   caBundle: <base64-encoded-ca-bundle>

您可以使用以下命令获取 caBundle：

kubectl get secret otelcol-injector-tls -n admission-control -o jsonpath='{.data.tls\.crt}' | base64 --decode

通过这些修改，您的 webhook 将使用 Kubernetes Secret 中存储的证书，而不是直接引用文件系统中的证书文件。这种方法更安全，也更符合 Kubernetes 的最佳实践。

记住，证书的更新和轮换需要手动管理。如果您希望自动化这个过程，可以考虑使用 cert-manager 这样的工具，它可以与 GKE 很好地集成，并自动处理证书的创建和更新。

zking2000 commented 4 weeks ago

!/bin/bash

set -e

Set variables

NAMESPACE="grafana-stack" DEST_NAMESPACE="injection-payload" SERVICE="otelcol-injector" SECRET="otelcol-injector-secret" WEBHOOK_NAME="otelcol-injection-webhook" CSR_NAME="${SERVICE}.${NAMESPACE}" CERT_DIR="${TMP_DIR}/certs" CONFIGMAP_NAME="otelcol-config" SERVICE_ACCOUNT="pft-uk-grafana-gke-sa"

export HTTPS_PROXY=http://localhost:8888

Function to check if resource exists and delete if it does

delete_if_exists() { local resource_type=$1 local resource_name=$2 local extra_args=$3

if kubectl get $resource_type $resource_name $extra_args &> /dev/null; then
    echo "Deleting $resource_type $resource_name"
    kubectl delete $resource_type $resource_name $extra_args
else
    echo "$resource_type $resource_name not found, skipping deletion"
fi

}

Cleanup approved csr

delete_if_exists CertificateSigningRequest ${CSR_NAME}

Delete deployment

if [ -f deployment.yaml ]; then echo "Deleting resources defined in deployment.yaml" kubectl delete -f deployment.yaml || echo "Failed to delete some resources from deployment.yaml" else echo "deployment.yaml not found, skipping" fi

Delete namespace

delete_if_exists namespace ${DEST_NAMESPACE}

Delete MutatingWebhookConfiguration

delete_if_exists MutatingWebhookConfiguration ${WEBHOOK_NAME}

Delete ClusterRoleBinding

delete_if_exists ClusterRoleBinding otelcol-sidecar-namespace-reader

Delete ClusterRole

delete_if_exists ClusterRole namespace-reader

echo "Cleanup completed."