search

openshift / origin

Conformance test suite for OpenShift
http://www.openshift.org
Apache License 2.0
8.48k stars 4.7k forks source link

Privileged pod cannot write to hostPath volume #26090

Closed chinazj closed 3 years ago

chinazj commented 3 years ago

[provide a description of the issue]

Version
[root@ocp-svc audit]# oc version
Client Version: 4.5.6
Server Version: 4.5.6
Kubernetes Version: v1.18.3+002a51f
Steps To Reproduce

experiment message

project = qegis
pod = aegis-manager****
serviceaccount = default

experiment debug: in bootstrap

[root@ocp-svc audit]# oc project qegis
Already on project "qegis" on server "https://api.lab.ocp.lan:6443".
[root@ocp-svc audit]# oc adm policy add-scc-to-user privileged -z default
clusterrole.rbac.authorization.k8s.io/system:openshift:scc:privileged added: "default"
[root@ocp-svc audit]#
[root@ocp-svc audit]# kubectl delete pod aegis-manager-864dc45559-4gnj8
pod "aegis-manager-864dc45559-4gnj8" deleted
[root@ocp-svc audit]# oc  get pod -w
NAME                              READY   STATUS                  RESTARTS   AGE
aegis-agent-hjsfm                 1/1     Running                 0          2d
aegis-agent-mrknt                 1/1     Running                 0          2d
aegis-discover-6f956d8f86-x8xt8   0/1     Init:CrashLoopBackOff   42         3h13m
aegis-elasticsearch-0             1/1     Running                 0          46h
aegis-es-head-97ddb5fd-2pvlr      1/1     Running                 0          46h
aegis-kafka-0                     1/1     Running                 0          46h
aegis-manager-864dc45559-2mjml    0/1     ContainerCreating       0          67s
aegis-mysql-0                     1/1     Running                 0          46h
aegis-redis-0                     1/1     Running                 0          46h
aegis-siddhi-84b8bc68b8-khwbp     0/1     Init:CrashLoopBackOff   42         3h13m
aegis-zookeeper-0                 1/1     Running                 0          46h
aegis-manager-864dc45559-2mjml    1/1     Running                 0          2m10s
^C[root@ocp-svc audit]#
[root@ocp-svc audit]# oc describe pod aegis-manager-864dc45559-2mjml
Name:         aegis-manager-864dc45559-2mjml
Namespace:    qegis
Priority:     0
Node:         ocp-w-1.lab.ocp.lan/192.168.22.211
Start Time:   Tue, 20 Apr 2021 14:09:38 +0800
Labels:       aegis-manager=true
              aegis-processor=true
              app=aegis-manager
              pod-template-hash=864dc45559
Annotations:  k8s.v1.cni.cncf.io/network-status:
                [{
                    "name": "openshift-sdn",
                    "interface": "eth0",
                    "ips": [
                        "10.131.0.28"
                    ],
                    "default": true,
                    "dns": {}
                }]
              k8s.v1.cni.cncf.io/networks-status:
                [{
                    "name": "openshift-sdn",
                    "interface": "eth0",
                    "ips": [
                        "10.131.0.28"
                    ],
                    "default": true,
                    "dns": {}
                }]
              openshift.io/scc: privileged
Status:       Running
IP:           10.131.0.28
IPs:
  IP:           10.131.0.28
Controlled By:  ReplicaSet/aegis-manager-864dc45559
Containers:
  aegis-manager:
    Container ID:  cri-o://e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107
    Image:         192.168.101.125/qegis/aegis-manager:3.1
    Image ID:      192.168.101.125/qegis/aegis-manager@sha256:2fdd41760dc06b205fe00543156cd83eba3db54ee2eff48ceea676b0dc2adc67
    Ports:         8081/TCP, 8443/TCP
    Host Ports:    0/TCP, 0/TCP
    Command:
      /bin/bash
      -c
      sleep 3600
    State:          Running
      Started:      Tue, 20 Apr 2021 14:11:45 +0800
    Ready:          True
    Restart Count:  0
    Limits:
      cpu:     4
      memory:  4Gi
    Requests:
      cpu:     1
      memory:  1Gi
    Environment:
      AEGIS_MANAGER_NODE_IP:   (v1:status.hostIP)
    Mounts:
      /opt/aegis-manager/config/application-prod.properties from aegis-manager-prod-config (rw,path="application-prod.properties")
      /opt/aegis-manager/logs from log-path (rw)
      /var/run/secrets/kubernetes.io/serviceaccount from default-token-q8r85 (ro)
Conditions:
  Type              Status
  Initialized       True
  Ready             True
  ContainersReady   True
  PodScheduled      True
Volumes:
  aegis-manager-prod-config:
    Type:      ConfigMap (a volume populated by a ConfigMap)
    Name:      aegis-manager-config
    Optional:  false
  log-path:
    Type:          HostPath (bare host directory volume)
    Path:          /var/log/aegis/manager
    HostPathType:
  default-token-q8r85:
    Type:        Secret (a volume populated by a Secret)
    SecretName:  default-token-q8r85
    Optional:    false
QoS Class:       Burstable
Node-Selectors:  aegis-server=true
Tolerations:     aegis-check=aegis-only:NoExecute
                 node.kubernetes.io/memory-pressure:NoSchedule
                 node.kubernetes.io/not-ready:NoExecute for 300s
                 node.kubernetes.io/unreachable:NoExecute for 300s
Events:
  Type    Reason          Age        From                          Message
  ----    ------          ----       ----                          -------
  Normal  Scheduled       <unknown>  default-scheduler             Successfully assigned qegis/aegis-manager-864dc45559-2mjml to ocp-w-1.lab.ocp.lan
  Normal  AddedInterface  <invalid>  multus                        Add eth0 [10.131.0.28/23]
  Normal  Pulling         <invalid>  kubelet, ocp-w-1.lab.ocp.lan  Pulling image "192.168.101.125/qegis/aegis-manager:3.1"
  Normal  Pulled          <invalid>  kubelet, ocp-w-1.lab.ocp.lan  Successfully pulled image "192.168.101.125/qegis/aegis-manager:3.1"
  Normal  Created         <invalid>  kubelet, ocp-w-1.lab.ocp.lan  Created container aegis-manager
  Normal  Started         <invalid>  kubelet, ocp-w-1.lab.ocp.lan  Started container aegis-manager
[root@ocp-svc audit]# oc get scc
NAME               PRIV    CAPS         SELINUX     RUNASUSER          FSGROUP     SUPGROUP    PRIORITY     READONLYROOTFS   VOLUMES
anyuid             false   <no value>   MustRunAs   RunAsAny           RunAsAny    RunAsAny    10           false            [configMap downwardAPI emptyDir persistentVolumeClaim projected secret]
hostaccess         false   <no value>   MustRunAs   MustRunAsRange     MustRunAs   RunAsAny    <no value>   false            [configMap downwardAPI emptyDir hostPath persistentVolumeClaim projected secret]
hostmount-anyuid   false   <no value>   MustRunAs   RunAsAny           RunAsAny    RunAsAny    <no value>   false            [configMap downwardAPI emptyDir hostPath nfs persistentVolumeClaim projected secret]
hostnetwork        false   <no value>   MustRunAs   MustRunAsRange     MustRunAs   MustRunAs   <no value>   false            [configMap downwardAPI emptyDir persistentVolumeClaim projected secret]
node-exporter      true    <no value>   RunAsAny    RunAsAny           RunAsAny    RunAsAny    <no value>   false            [*]
nonroot            false   <no value>   MustRunAs   MustRunAsNonRoot   RunAsAny    RunAsAny    <no value>   false            [configMap downwardAPI emptyDir persistentVolumeClaim projected secret]
privileged         true    [*]          RunAsAny    RunAsAny           RunAsAny    RunAsAny    <no value>   false            [*]
qegis-manager      true    <no value>   RunAsAny    RunAsAny           RunAsAny    RunAsAny    <no value>   false            [*]
restricted         false   <no value>   MustRunAs   MustRunAsRange     MustRunAs   RunAsAny    <no value>   false            [configMap downwardAPI emptyDir persistentVolumeClaim projected secret]
[root@ocp-svc audit]# oc get scc privileged -oyaml
allowHostDirVolumePlugin: true
allowHostIPC: true
allowHostNetwork: true
allowHostPID: true
allowHostPorts: true
allowPrivilegeEscalation: true
allowPrivilegedContainer: true
allowedCapabilities:
- '*'
allowedUnsafeSysctls:
- '*'
apiVersion: security.openshift.io/v1
defaultAddCapabilities: null
fsGroup:
  type: RunAsAny
groups:
- system:cluster-admins
- system:nodes
- system:masters
kind: SecurityContextConstraints
metadata:
  annotations:
    kubernetes.io/description: 'privileged allows access to all privileged and host
      features and the ability to run as any user, any group, any fsGroup, and with
      any SELinux context.  WARNING: this is the most relaxed SCC and should be used
      only for cluster administration. Grant with caution.'
    release.openshift.io/create-only: "true"
  creationTimestamp: "2021-04-15T04:15:05Z"
  generation: 1
  managedFields:
  - apiVersion: security.openshift.io/v1
    fieldsType: FieldsV1
    fieldsV1:
      f:allowHostDirVolumePlugin: {}
      f:allowHostIPC: {}
      f:allowHostNetwork: {}
      f:allowHostPID: {}
      f:allowHostPorts: {}
      f:allowPrivilegeEscalation: {}
      f:allowPrivilegedContainer: {}
      f:allowedCapabilities: {}
      f:allowedUnsafeSysctls: {}
      f:defaultAddCapabilities: {}
      f:fsGroup:
        .: {}
        f:type: {}
      f:groups: {}
      f:metadata:
        f:annotations:
          .: {}
          f:kubernetes.io/description: {}
          f:release.openshift.io/create-only: {}
      f:priority: {}
      f:readOnlyRootFilesystem: {}
      f:requiredDropCapabilities: {}
      f:runAsUser:
        .: {}
        f:type: {}
      f:seLinuxContext:
        .: {}
        f:type: {}
      f:seccompProfiles: {}
      f:supplementalGroups:
        .: {}
        f:type: {}
      f:users: {}
      f:volumes: {}
    manager: cluster-version-operator
    operation: Update
    time: "2021-04-15T04:15:05Z"
  name: privileged
  resourceVersion: "984"
  selfLink: /apis/security.openshift.io/v1/securitycontextconstraints/privileged
  uid: 0b908c8d-b10e-4e80-be0f-7b1c70030f5a
priority: null
readOnlyRootFilesystem: false
requiredDropCapabilities: null
runAsUser:
  type: RunAsAny
seLinuxContext:
  type: RunAsAny
seccompProfiles:
- '*'
supplementalGroups:
  type: RunAsAny
users:
- system:admin
- system:serviceaccount:openshift-infra:build-controller
volumes:
- '*'
[root@ocp-svc audit]#
[root@ocp-svc audit]# oc get project qegis -oyaml
apiVersion: project.openshift.io/v1
kind: Project
metadata:
  annotations:
    openshift.io/sa.scc.mcs: s0:c24,c9
    openshift.io/sa.scc.supplemental-groups: 1000570000/10000
    openshift.io/sa.scc.uid-range: 1000570000/10000
  creationTimestamp: "2021-04-18T03:55:18Z"
  labels:
    name: qegis
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:labels:
          .: {}
          f:name: {}
      f:status:
        f:phase: {}
    manager: Go-http-client
    operation: Update
    time: "2021-04-18T03:55:18Z"
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:annotations:
          .: {}
          f:openshift.io/sa.scc.mcs: {}
          f:openshift.io/sa.scc.supplemental-groups: {}
          f:openshift.io/sa.scc.uid-range: {}
    manager: cluster-policy-controller
    operation: Update
    time: "2021-04-18T03:55:21Z"
  name: qegis
  resourceVersion: "1313383"
  selfLink: /apis/project.openshift.io/v1/projects/qegis
  uid: 116a77bd-5a73-4e26-b26f-27d55a3dd044
spec:
  finalizers:
  - kubernetes
status:
  phase: Active
[root@ocp-svc audit]# oc exec -it aegis-manager-864dc45559-2mjml sh
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl kubectl exec [POD] -- [COMMAND] instead.
/opt/aegis-manager $
/opt/aegis-manager $
/opt/aegis-manager $ ls
aegis-manager.jar  config             logs               static
bin                lib                manager_version    tools
/opt/aegis-manager $ ls logs/
ls: can't open 'logs/': Permission denied
/opt/aegis-manager $ whoami
11111
/opt/aegis-manager $ exit

in node

[core@ocp-w-1 ~]$ ls -Z /var/log/aegis
system_u:object_r:container_log_t:s0 agent     system_u:object_r:container_log_t:s0 elasticsearch
system_u:object_r:container_log_t:s0 discover  system_u:object_r:container_log_t:s0 manager
[core@ocp-w-1 ~]$ ls -Z /var/log/
  system_u:object_r:container_log_t:s0 aegis                system_u:object_r:var_log_t:s0 private
     system_u:object_r:auditd_log_t:s0 audit              system_u:object_r:samba_log_t:s0 samba
        system_u:object_r:faillog_t:s0 btmp            system_u:object_r:sssd_var_log_t:s0 sssd
system_u:object_r:chronyd_var_log_t:s0 chrony               system_u:object_r:var_log_t:s0 vmware-network.1.log
        system_u:object_r:var_log_t:s0 containers           system_u:object_r:var_log_t:s0 vmware-network.2.log
  system_u:object_r:container_log_t:s0 crio                 system_u:object_r:var_log_t:s0 vmware-network.log
   system_u:object_r:glusterd_log_t:s0 glusterfs         system_u:object_r:vmware_log_t:s0 vmware-vgauthsvc.log.0
        system_u:object_r:var_log_t:s0 journal           system_u:object_r:vmware_log_t:s0 vmware-vmsvc-root.log
        system_u:object_r:lastlog_t:s0 lastlog           system_u:object_r:vmware_log_t:s0 vmware-vmtoolsd-root.log
system_u:object_r:openvswitch_log_t:s0 openvswitch             system_u:object_r:wtmp_t:s0 wtmp
        system_u:object_r:var_log_t:s0 pods
[core@ocp-w-1 ~]$ getenforce ^C
[core@ocp-w-1 ~]$ sudo crictl ps | grep manager
e74cd9ce6d8e5       192.168.101.125/qegis/aegis-manager@sha256:2fdd41760dc06b205fe00543156cd83eba3db54ee2eff48ceea676b0dc2adc67         9 minutes ago       Running             aegis-manager                0                   0b33913a34e34
[core@ocp-w-1 ~]$ ps -ef | grep e74cd9ce6d8e5
root     3282846       1  0 06:11 ?        00:00:00 /usr/libexec/crio/conmon -s -c e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107 -n k8s_aegis-manager_aegis-manager-864dc45559-2mjml_qegis_0dfc7c8e-c833-41a7-88d4-1fb5533280dd_0 -u e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107 -r /usr/bin/runc -b /var/run/containers/storage/overlay-containers/e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107/userdata --persist-dir /var/lib/containers/storage/overlay-containers/e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107/userdata -p /var/run/containers/storage/overlay-containers/e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107/userdata/pidfile -P /var/run/containers/storage/overlay-containers/e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107/userdata/conmon-pidfile -l /var/log/pods/qegis_aegis-manager-864dc45559-2mjml_0dfc7c8e-c833-41a7-88d4-1fb5533280dd/aegis-manager/0.log --exit-dir /var/run/crio/exits --socket-dir-path /var/run/crio --log-level info --runtime-arg --root=/run/runc
core     3324605 3318157  1 06:22 pts/0    00:00:00 grep --color=auto e74cd9ce6d8e5
[core@ocp-w-1 ~]$ ps -ef | grep 3282846
root     3282846       1  0 06:11 ?        00:00:00 /usr/libexec/crio/conmon -s -c e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107 -n k8s_aegis-manager_aegis-manager-864dc45559-2mjml_qegis_0dfc7c8e-c833-41a7-88d4-1fb5533280dd_0 -u e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107 -r /usr/bin/runc -b /var/run/containers/storage/overlay-containers/e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107/userdata --persist-dir /var/lib/containers/storage/overlay-containers/e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107/userdata -p /var/run/containers/storage/overlay-containers/e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107/userdata/pidfile -P /var/run/containers/storage/overlay-containers/e74cd9ce6d8e523b45d99f1d11f25244e3d8f6d92b6bb762a30b936272e54107/userdata/conmon-pidfile -l /var/log/pods/qegis_aegis-manager-864dc45559-2mjml_0dfc7c8e-c833-41a7-88d4-1fb5533280dd/aegis-manager/0.log --exit-dir /var/run/crio/exits --socket-dir-path /var/run/crio --log-level info --runtime-arg --root=/run/runc
11111    3282907 3282846  0 06:11 ?        00:00:00 sleep 3600
core     3325555 3318157  0 06:22 pts/0    00:00:00 grep --color=auto 3282846
[core@ocp-w-1 ~]$ cat /proc/3282907/status
Name:   sleep
Umask:  0022
State:  S (sleeping)
Tgid:   3282907
Ngid:   0
Pid:    3282907
PPid:   3282846
TracerPid:  0
Uid:    11111   11111   11111   11111
Gid:    0   0   0   0
FDSize: 64
Groups: 11111
NStgid: 3282907 1
NSpid:  3282907 1
NSpgid: 3282907 1
NSsid:  3282907 1
VmPeak:     1520 kB
VmSize:     1520 kB
VmLck:         0 kB
VmPin:         0 kB
VmHWM:         4 kB
VmRSS:         4 kB
RssAnon:           4 kB
RssFile:           0 kB
RssShmem:          0 kB
VmData:       20 kB
VmStk:       132 kB
VmExe:       636 kB
VmLib:       236 kB
VmPTE:        40 kB
VmSwap:        0 kB
HugetlbPages:          0 kB
CoreDumping:    0
Threads:    1
SigQ:   0/96072
SigPnd: 0000000000000000
ShdPnd: 0000000000000000
SigBlk: 0000000000000000
SigIgn: 0000000000000000
SigCgt: 0000000000000000
CapInh: 00000000000425fb
CapPrm: 0000000000000000
CapEff: 0000000000000000
CapBnd: 00000000000425fb
CapAmb: 0000000000000000
NoNewPrivs: 0
Seccomp:    0
Speculation_Store_Bypass:   vulnerable
Cpus_allowed:   fff
Cpus_allowed_list:  0-11
Mems_allowed:   00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000000,00000003
Mems_allowed_list:  0-1
voluntary_ctxt_switches:    77
nonvoluntary_ctxt_switches: 4
Current Result

Permission denied. when Selinux is permissive.its ok

chinazj commented 3 years ago

deployment

apiVersion: apps/v1
kind: Deployment
metadata:
  name: aegis-manager
  namespace: qegis
  labels:
    app: aegis-manager
spec:
  selector:
    matchLabels:
      app: aegis-manager
  template:
    metadata:
      labels:
        app: aegis-manager
        aegis-manager: "true"
        aegis-processor: "true"
    spec:
      nodeSelector:
        aegis-server: "true"
      tolerations:
        - key: "aegis-check"
          operator: "Equal"
          value: "aegis-only"
          effect: "NoExecute"
      securityContext:
        fsGroup: 11111
        runAsUser: 11111
      containers:
        - name: aegis-manager
          image: 192.168.101.125/qegis/aegis-manager:3.1
          imagePullPolicy: Always
          resources:
            limits:
              cpu: 4000m
              memory: 4Gi
            requests:
              cpu: 1000m
              memory: 1Gi
          command: ["/bin/bash", "-c", "sleep 3600" ]
          ports:
            - containerPort: 8081
              name: http-port
            - containerPort: 8443
              name: https-port
          env:
            - name: AEGIS_MANAGER_NODE_IP
              valueFrom:
                fieldRef:
                  fieldPath: status.hostIP
          volumeMounts:
            - name: aegis-manager-prod-config
              mountPath: /opt/aegis-manager/config/application-prod.properties
              subPath: application-prod.properties
            - name: log-path
              mountPath: /opt/aegis-manager/logs
      volumes:
        - name: aegis-manager-prod-config
          configMap:
            name: aegis-manager-config
            items:
              - key: application-prod.properties
                path: application-prod.properties
        - name: log-path
          hostPath:
            path: /var/log/aegis/manager