search

canonical / lxd

Powerful system container and virtual machine manager
https://canonical.com/lxd
GNU Affero General Public License v3.0
4.38k stars 931 forks source link

Problems to run podman containers inside LXD container #10389

Closed lucasbaile closed 2 years ago

lucasbaile commented 2 years ago

Required information

config:
  core.https_address: '[::]:8443'
api_extensions:
- storage_zfs_remove_snapshots
- container_host_shutdown_timeout
- container_stop_priority
- container_syscall_filtering
- auth_pki
- container_last_used_at
- etag
- patch
- usb_devices
- https_allowed_credentials
- image_compression_algorithm
- directory_manipulation
- container_cpu_time
- storage_zfs_use_refquota
- storage_lvm_mount_options
- network
- profile_usedby
- container_push
- container_exec_recording
- certificate_update
- container_exec_signal_handling
- gpu_devices
- container_image_properties
- migration_progress
- id_map
- network_firewall_filtering
- network_routes
- storage
- file_delete
- file_append
- network_dhcp_expiry
- storage_lvm_vg_rename
- storage_lvm_thinpool_rename
- network_vlan
- image_create_aliases
- container_stateless_copy
- container_only_migration
- storage_zfs_clone_copy
- unix_device_rename
- storage_lvm_use_thinpool
- storage_rsync_bwlimit
- network_vxlan_interface
- storage_btrfs_mount_options
- entity_description
- image_force_refresh
- storage_lvm_lv_resizing
- id_map_base
- file_symlinks
- container_push_target
- network_vlan_physical
- storage_images_delete
- container_edit_metadata
- container_snapshot_stateful_migration
- storage_driver_ceph
- storage_ceph_user_name
- resource_limits
- storage_volatile_initial_source
- storage_ceph_force_osd_reuse
- storage_block_filesystem_btrfs
- resources
- kernel_limits
- storage_api_volume_rename
- macaroon_authentication
- network_sriov
- console
- restrict_devlxd
- migration_pre_copy
- infiniband
- maas_network
- devlxd_events
- proxy
- network_dhcp_gateway
- file_get_symlink
- network_leases
- unix_device_hotplug
- storage_api_local_volume_handling
- operation_description
- clustering
- event_lifecycle
- storage_api_remote_volume_handling
- nvidia_runtime
- container_mount_propagation
- container_backup
- devlxd_images
- container_local_cross_pool_handling
- proxy_unix
- proxy_udp
- clustering_join
- proxy_tcp_udp_multi_port_handling
- network_state
- proxy_unix_dac_properties
- container_protection_delete
- unix_priv_drop
- pprof_http
- proxy_haproxy_protocol
- network_hwaddr
- proxy_nat
- network_nat_order
- container_full
- candid_authentication
- backup_compression
- candid_config
- nvidia_runtime_config
- storage_api_volume_snapshots
- storage_unmapped
- projects
- candid_config_key
- network_vxlan_ttl
- container_incremental_copy
- usb_optional_vendorid
- snapshot_scheduling
- snapshot_schedule_aliases
- container_copy_project
- clustering_server_address
- clustering_image_replication
- container_protection_shift
- snapshot_expiry
- container_backup_override_pool
- snapshot_expiry_creation
- network_leases_location
- resources_cpu_socket
- resources_gpu
- resources_numa
- kernel_features
- id_map_current
- event_location
- storage_api_remote_volume_snapshots
- network_nat_address
- container_nic_routes
- rbac
- cluster_internal_copy
- seccomp_notify
- lxc_features
- container_nic_ipvlan
- network_vlan_sriov
- storage_cephfs
- container_nic_ipfilter
- resources_v2
- container_exec_user_group_cwd
- container_syscall_intercept
- container_disk_shift
- storage_shifted
- resources_infiniband
- daemon_storage
- instances
- image_types
- resources_disk_sata
- clustering_roles
- images_expiry
- resources_network_firmware
- backup_compression_algorithm
- ceph_data_pool_name
- container_syscall_intercept_mount
- compression_squashfs
- container_raw_mount
- container_nic_routed
- container_syscall_intercept_mount_fuse
- container_disk_ceph
- virtual-machines
- image_profiles
- clustering_architecture
- resources_disk_id
- storage_lvm_stripes
- vm_boot_priority
- unix_hotplug_devices
- api_filtering
- instance_nic_network
- clustering_sizing
- firewall_driver
- projects_limits
- container_syscall_intercept_hugetlbfs
- limits_hugepages
- container_nic_routed_gateway
- projects_restrictions
- custom_volume_snapshot_expiry
- volume_snapshot_scheduling
- trust_ca_certificates
- snapshot_disk_usage
- clustering_edit_roles
- container_nic_routed_host_address
- container_nic_ipvlan_gateway
- resources_usb_pci
- resources_cpu_threads_numa
- resources_cpu_core_die
- api_os
- container_nic_routed_host_table
- container_nic_ipvlan_host_table
- container_nic_ipvlan_mode
- resources_system
- images_push_relay
- network_dns_search
- container_nic_routed_limits
- instance_nic_bridged_vlan
- network_state_bond_bridge
- usedby_consistency
- custom_block_volumes
- clustering_failure_domains
- resources_gpu_mdev
- console_vga_type
- projects_limits_disk
- network_type_macvlan
- network_type_sriov
- container_syscall_intercept_bpf_devices
- network_type_ovn
- projects_networks
- projects_networks_restricted_uplinks
- custom_volume_backup
- backup_override_name
- storage_rsync_compression
- network_type_physical
- network_ovn_external_subnets
- network_ovn_nat
- network_ovn_external_routes_remove
- tpm_device_type
- storage_zfs_clone_copy_rebase
- gpu_mdev
- resources_pci_iommu
- resources_network_usb
- resources_disk_address
- network_physical_ovn_ingress_mode
- network_ovn_dhcp
- network_physical_routes_anycast
- projects_limits_instances
- network_state_vlan
- instance_nic_bridged_port_isolation
- instance_bulk_state_change
- network_gvrp
- instance_pool_move
- gpu_sriov
- pci_device_type
- storage_volume_state
- network_acl
- migration_stateful
- disk_state_quota
- storage_ceph_features
- projects_compression
- projects_images_remote_cache_expiry
- certificate_project
- network_ovn_acl
- projects_images_auto_update
- projects_restricted_cluster_target
- images_default_architecture
- network_ovn_acl_defaults
- gpu_mig
- project_usage
- network_bridge_acl
- warnings
- projects_restricted_backups_and_snapshots
- clustering_join_token
- clustering_description
- server_trusted_proxy
- clustering_update_cert
- storage_api_project
- server_instance_driver_operational
- server_supported_storage_drivers
- event_lifecycle_requestor_address
- resources_gpu_usb
- clustering_evacuation
- network_ovn_nat_address
- network_bgp
- network_forward
- custom_volume_refresh
- network_counters_errors_dropped
- metrics
- image_source_project
- clustering_config
- network_peer
- linux_sysctl
- network_dns
- ovn_nic_acceleration
- certificate_self_renewal
- instance_project_move
- storage_volume_project_move
- cloud_init
- network_dns_nat
- database_leader
- instance_all_projects
- clustering_groups
- ceph_rbd_du
- instance_get_full
- qemu_metrics
- gpu_mig_uuid
- event_project
- clustering_evacuation_live
- instance_allow_inconsistent_copy
- network_state_ovn
- storage_volume_api_filtering
- image_restrictions
- storage_zfs_export
- network_dns_records
- storage_zfs_reserve_space
- network_acl_log
- storage_zfs_blocksize
- metrics_cpu_seconds
- instance_snapshot_never
- certificate_token
- instance_nic_routed_neighbor_probe
- event_hub
- agent_nic_config
- projects_restricted_intercept
- metrics_authentication
api_status: stable
api_version: "1.0"
auth: trusted
public: false
auth_methods:
- tls
environment:
  addresses:
  - 136.243.44.22:8443
  - '[2a01:4f8:212:c0d::2]:8443'
  - 10.27.217.1:8443
  architectures:
  - x86_64
  - i686
  certificate: |
    -----BEGIN CERTIFICATE-----
    ...
    -----END CERTIFICATE-----
  certificate_fingerprint: ...
  driver: lxc | qemu
  driver_version: 4.0.12 | 6.1.1
  firewall: xtables
  kernel: Linux
  kernel_architecture: x86_64
  kernel_features:
    idmapped_mounts: "false"
    netnsid_getifaddrs: "true"
    seccomp_listener: "false"
    seccomp_listener_continue: "false"
    shiftfs: "false"
    uevent_injection: "true"
    unpriv_fscaps: "true"
  kernel_version: 4.18.0-348.2.1.el8_5.x86_64
  lxc_features:
    cgroup2: "true"
    core_scheduling: "true"
    devpts_fd: "true"
    idmapped_mounts_v2: "true"
    mount_injection_file: "true"
    network_gateway_device_route: "true"
    network_ipvlan: "true"
    network_l2proxy: "true"
    network_phys_macvlan_mtu: "true"
    network_veth_router: "true"
    pidfd: "true"
    seccomp_allow_deny_syntax: "true"
    seccomp_notify: "true"
    seccomp_proxy_send_notify_fd: "true"
  os_name: CentOS Stream
  os_version: "8"
  project: default
  server: lxd
  server_clustered: false
  server_event_mode: full-mesh
  server_name: ti-staging
  server_pid: 401737
  server_version: "4.24"
  storage: lvm
  storage_version: 2.03.07(2) (2019-11-30) / 1.02.167 (2019-11-30) / 4.43.0
  storage_supported_drivers:
  - name: ceph
    version: 15.2.14
    remote: true
  - name: btrfs
    version: 5.4.1
    remote: false
  - name: cephfs
    version: 15.2.14
    remote: true
  - name: dir
    version: "1"
    remote: false
  - name: lvm
    version: 2.03.07(2) (2019-11-30) / 1.02.167 (2019-11-30) / 4.43.0
    remote: false

```bash
$lxc storage show default
config:
  lvm.thinpool_name: LXDThinPool
  lvm.vg_name: default
  size: 750GB
  source: /var/snap/lxd/common/lxd/disks/default.img
  volume.size: 20GB
description: ""
name: default
driver: lvm
used_by:
- /1.0/images/b06d81d479c7d52dc4cc978b114a67b42a99ca498db28bf89b8aee1f80364f77
- /1.0/instances/pr7049
- /1.0/instances/promoted-bunny
- /1.0/profiles/default
status: Created
locations:
- none

Issue description

So, I understand the question is a little bit convoluted, but for reasons regarding my CI/CD infrastructure, I'm currently in a situation where I want to build and run podman images inside LXD containers. I'm not trying to use podman on rootless mode, so every podman command is ran as sudo. Running a fresh LXD container using the base image images:centos/8-Stream, I install podman and try pulling the quay.io/centos/centos:stream8 podman image, and this pull fails if the current error message:

$ podman pull quay.io/centos/centos:stream8
Trying to pull quay.io/centos/centos:stream8...
Getting image source signatures
Copying blob a0b8f3931ffa skipped: already exists  
Copying blob 1ac891d08dc2 done  
Copying blob 4a7e61ebcfec done  
Copying blob 04f0eb705bff done  
Error: writing blob: adding layer with blob "sha256:04f0eb705bffc1db22f04bd42987ebb9d5e40c08c0253d0c2a56881c75bc6af8": Error processing tar file(exit status 1): operation not permitted

I am aware of this fix/workaround:

Steps to reproduce

  1. Launch a fresh LXD container using the images:centos/8-Stream base image: 
    lxc launch images:centos/8-Stream testcontainer
  2. From inside the LXD container, simply install podman and try pulling the quay.io/centos/centos:stream8 image: 
    lxc exec testcontainer -- bash
dnf update -y && dnf install podman -y
podman pull quay.io/centos/centos:stream8

Information to attach

Resources: Processes: 12 Disk usage: root: 1.14GiB CPU usage: CPU usage (in seconds): 25 Memory usage: Memory (current): 617.56MiB Memory (peak): 889.88MiB Network usage: eth0: Type: broadcast State: UP Host interface: veth4f3a3b2d MAC address: 00:16:3e:d8:af:b3 MTU: 1500 Bytes received: 351.33MB Bytes sent: 2.41MB Packets received: 178804 Packets sent: 34966 IP addresses: inet: 10.27.217.72/24 (global) inet6: fe80::216:3eff:fed8:afb3/64 (link) lo: Type: loopback State: UP MTU: 65536 Bytes received: 0B Bytes sent: 0B Packets received: 0 Packets sent: 0 IP addresses: inet: 127.0.0.1/8 (local) inet6: ::1/128 (local)

Log:

lxc testcontainer 20220510165502.672 ERROR utils - utils.c:lxc_can_use_pidfd:1792 - Kernel does not support pidfds lxc testcontainer 20220510165502.672 WARN conf - conf.c:lxc_map_ids:3592 - newuidmap binary is missing lxc testcontainer 20220510165502.672 WARN conf - conf.c:lxc_map_ids:3598 - newgidmap binary is missing lxc testcontainer 20220510165502.673 WARN conf - conf.c:lxc_map_ids:3592 - newuidmap binary is missing lxc testcontainer 20220510165502.673 WARN conf - conf.c:lxc_map_ids:3598 - newgidmap binary is missing lxc testcontainer 20220510165509.488 WARN attach - attach.c:get_attach_context:477 - No security context received

 - [x] Container configuration (`lxc config show NAME --expanded`)
 ```yaml
 architecture: x86_64
config:
  boot.autostart: "false"
  image.architecture: amd64
  image.description: Centos 8-Stream amd64 (20220510_07:08)
  image.os: Centos
  image.release: 8-Stream
  image.serial: "20220510_07:08"
  image.type: squashfs
  image.variant: default
  volatile.base_image: b06d81d479c7d52dc4cc978b114a67b42a99ca498db28bf89b8aee1f80364f77
  volatile.eth0.host_name: veth4f3a3b2d
  volatile.eth0.hwaddr: 00:16:3e:d8:af:b3
  volatile.idmap.base: "0"
  volatile.idmap.current: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000000000},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000000000}]'
  volatile.idmap.next: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000000000},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000000000}]'
  volatile.last_state.idmap: '[{"Isuid":true,"Isgid":false,"Hostid":1000000,"Nsid":0,"Maprange":1000000000},{"Isuid":false,"Isgid":true,"Hostid":1000000,"Nsid":0,"Maprange":1000000000}]'
  volatile.last_state.power: RUNNING
  volatile.uuid: 9981e56f-176d-40d3-aa40-dfedc3b8a614
devices:
  eth0:
    name: eth0
    network: lxdbr0
    type: nic
  root:
    path: /
    pool: default
    type: disk
ephemeral: false
profiles:
- default
stateful: false
description: ""
simondeziel commented 2 years ago

I tried reproducing it but it works for me on:

$ uname -a
Linux sdeziel-lemur 5.13.0-40-generic #45~20.04.1-Ubuntu SMP Mon Apr 4 09:38:31 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux

$ snap list lxd
Name  Version      Rev    Tracking       Publisher   Notes
lxd   5.1-4ae3604  23001  latest/stable  canonical✓  -
[root@testcontainer ~]# podman pull quay.io/centos/centos:stream8
Error: 'overlay' is not supported over zfs, a mount_program is required: backing file system is unsupported for this graph driver
[root@testcontainer ~]# vi /etc/containers/storage.conf
[root@testcontainer ~]# podman pull quay.io/centos/centos:stream8
Trying to pull quay.io/centos/centos:stream8...
Getting image source signatures
Copying blob 4a7e61ebcfec done  
Copying blob a0b8f3931ffa done  
Copying blob 04f0eb705bff done  
Copying blob 1ac891d08dc2 done  
Copying config 67daa3fbd6 done  
Writing manifest to image destination
Storing signatures
67daa3fbd687e011241a14ec37dc733e19cccc35faf944a0455dc78dbddb9caf
[root@testcontainer ~]#
stgraber commented 2 years ago

Closing as this feels more like a support question (running a particular workload on LXD) rather than a bug in the container runtime itself. I'd suggest using https://discuss.linuxcontainers.org where we're also all present and is a better location for such questions as solutions tend to be more visible and better indexed than Github.