REST API: internal network forwards DNS requests to external nameservers

Issue Description

Docker recently changed how DNS requests are handled from internal networks. Adjust the Podman compatibility rest API to match the new behaviour of the Docker rest API.

Side note: Podman already has functionality for disabling DNS (see the podman network create command-line option --disable-dns).

Steps to reproduce the issue

Create the file /tmp/test.bashwith the file contents

#!/bin/bash

set -o errexit
set -o nounset

unixsocket=$1
testname=$2

curlverbose=""
# curlverbose="-v"

dir=$(mktemp -d)

cd "$dir"

cat <<EOF > Dockerfile
FROM docker.io/library/fedora:40
RUN dnf -y install bind-utils
CMD sh -c "host -a kernel.org > /resultdir/result 2>&1"
EOF

tar -cf Dockerfile.tar.gz Dockerfile 

echo ______________ build image
curl $curlverbose \
-s \
--unix-socket "$unixsocket" \
-X POST \
-H "Content-Type: application/x-tar" \
--data-binary @Dockerfile.tar.gz \
"http://localhost/v1.24/build?t=localhost/$testname"

echo ______________ create network
(
cat <<EOF
{
 "Name": "$testname",
 "Driver": "bridge",
 "Internal": true,
 "Attachable": false,
 "Ingress": false,
 "Options": {
 },
 "Labels": {
 }
}
EOF
) | jq . | curl $curlverbose \
       -s \
       --unix-socket "$unixsocket" \
       -X POST \
       -H "Content-Type: application/json" \
       -d @- \
       "http://localhost/v1.24/networks/create"

echo ______________ create container
(
cat <<EOF
{
 "Image": "localhost/$testname",
 "ExposedPorts": {
 },
 "HostConfig": {
 "NetworkMode" : "$testname",
   "binds" : [
 "$dir:/resultdir:Z"
   ],
   "AutoRemove": true
 }
}
EOF
) | jq . | curl $curlverbose \
       -s \
       --unix-socket "$unixsocket" \
       -X POST \
       -H "Content-Type: application/json" \
       -d @- \
       "http://localhost/v1.24/containers/create?name=$testname"

echo ______________ start container
curl $curlverbose \
-s \
--unix-socket "$unixsocket" \
-X POST \
"http://localhost/v1.24/containers/$testname/start"
echo ______________ wait for container
curl $curlverbose \
-s \
--unix-socket "$unixsocket" \
-X POST \
"http://localhost/v1.24/containers/$testname/wait"
echo ______________ output from container command:
cat $dir/result

sudo -i
systemctl start podman.socket
bash /tmp/test.bash /run/podman/podman.sock test1
useradd test
usermod -aG docker test
systemctl start docker.socket
machinectl shell --uid test
systemctl --user start podman.socket
bash /tmp/test.bash $XDG_RUNTIME_DIR/podman/podman.sock test2
bash /tmp/test.bash /var/run/docker.sock test3

Describe the results you received

Step 4 prints the following output

______________ build image
{"stream":"STEP 1/3: FROM docker.io/library/fedora:40\n"}
{"stream":"STEP 2/3: RUN dnf -y install bind-utils\n"}
{"stream":"--\u003e Using cache 7bb15fd261e9d671a21d495f543d022a47cce4a4577b486e7b3ba353d693d10e\n"}
{"stream":"--\u003e 7bb15fd261e9\n"}
{"stream":"STEP 3/3: CMD sh -c \"host -a kernel.org \u003e /resultdir/result 2\u003e\u00261\"\n"}
{"stream":"--\u003e Using cache 8b042593fd9204d296ba5aafc3caaed468aa1492e37eeca916e1d6e78603a76f\n"}
{"stream":"COMMIT localhost/test1\n"}
{"stream":"--\u003e 8b042593fd92\n"}
{"stream":"Successfully tagged localhost/test1:latest\n"}
{"stream":"Successfully tagged localhost/mytest64:latest\n"}
{"stream":"Successfully tagged localhost/mytest61:latest\n"}
{"stream":"8b042593fd9204d296ba5aafc3caaed468aa1492e37eeca916e1d6e78603a76f\n"}
{"aux":{"ID":"sha256:8b042593fd9204d296ba5aafc3caaed468aa1492e37eeca916e1d6e78603a76f"}}
{"stream":"Successfully built 8b042593fd92\n"}
{"stream":"Successfully tagged localhost/test1\n"}
______________ create network
{"Id":"d9ed3f8fd49648f71cf75a03838d1f74c6c059f10b66cdd0baa3e40ea251d8ee","Warning":""}
______________ create container
{"Id":"b9b4f5f39dbcd8a1a4110ef501f3258be51b33dce7c3462a1dea81e7f14a094a","Warnings":[]}
______________ start container
______________ wait for container
{"StatusCode":0,"Error":{"Message":"no container with ID b9b4f5f39dbcd8a1a4110ef501f3258be51b33dce7c3462a1dea81e7f14a094a found in database: no such container"}}
______________ output from container command:
Trying "kernel.org"
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 14454
;; flags: qr rd ra; QUERY: 1, ANSWER: 14, AUTHORITY: 0, ADDITIONAL: 0

;; QUESTION SECTION:
;kernel.org.            IN  ANY

;; ANSWER SECTION:
kernel.org.     86065   IN  SOA ns11.constellix.com. dns.constellix.com. 2015011869 43200 3600 1209600 180
kernel.org.     263 IN  AAAA    2604:1380:4641:c500::1
kernel.org.     263 IN  A   139.178.84.217
kernel.org.     1875    IN  NS  ns21.constellix.com.
kernel.org.     1875    IN  NS  ns61.constellix.net.
kernel.org.     1875    IN  NS  ns11.constellix.com.
kernel.org.     1875    IN  NS  ns51.constellix.net.
kernel.org.     1875    IN  NS  ns31.constellix.com.
kernel.org.     1875    IN  NS  ns41.constellix.net.
kernel.org.     265 IN  TXT "v=spf1 include:_spf.kernel.org ~all"
kernel.org.     265 IN  TXT "google-site-verification=nb4Wx5xBQM3b-a3OV2KSlPL786714Y7M6GfBtJf1opQ"
kernel.org.     263 IN  MX  10 smtp2.kernel.org.
kernel.org.     263 IN  MX  10 smtp3.kernel.org.
kernel.org.     263 IN  MX  10 smtp1.kernel.org.

Received 449 bytes from 10.89.4.1#53 in 22 ms

Step 10 prints the following output:

______________ build image
{"stream":"STEP 1/3: FROM docker.io/library/fedora:40\n"}
{"stream":"STEP 2/3: RUN dnf -y install bind-utils\n"}
{"stream":"--\u003e Using cache 99474c512320aca80ebd384df1b45c5447f733b259d88c11da2f49eaf5bdb27b\n"}
{"stream":"--\u003e 99474c512320\n"}
{"stream":"STEP 3/3: CMD sh -c \"host -a kernel.org \u003e /resultdir/result 2\u003e\u00261\"\n"}
{"stream":"--\u003e Using cache c6ddfc0ee3f2b9a758c0730a420a4bb5c028235870580d8eb9eba6f4019b21e7\n"}
{"stream":"COMMIT localhost/test2\n"}
{"stream":"--\u003e c6ddfc0ee3f2\n"}
{"stream":"Successfully tagged localhost/test2:latest\n"}
{"stream":"Successfully tagged localhost/mytest52:latest\n"}
{"stream":"Successfully tagged localhost/mytest50:latest\n"}
{"stream":"Successfully tagged localhost/mytest49:latest\n"}
{"stream":"Successfully tagged localhost/mytest46:latest\n"}
{"stream":"Successfully tagged localhost/mytest43:latest\n"}
{"stream":"Successfully tagged localhost/mytest42:latest\n"}
{"stream":"Successfully tagged localhost/mytest41:latest\n"}
{"stream":"Successfully tagged localhost/mytest40:latest\n"}
{"stream":"Successfully tagged localhost/mytest39:latest\n"}
{"stream":"Successfully tagged localhost/mytest38:latest\n"}
{"stream":"Successfully tagged localhost/mytest35:latest\n"}
{"stream":"c6ddfc0ee3f2b9a758c0730a420a4bb5c028235870580d8eb9eba6f4019b21e7\n"}
{"aux":{"ID":"sha256:c6ddfc0ee3f2b9a758c0730a420a4bb5c028235870580d8eb9eba6f4019b21e7"}}
{"stream":"Successfully built c6ddfc0ee3f2\n"}
{"stream":"Successfully tagged localhost/test2\n"}
______________ create network
{"Id":"44e381befef8afe9e173517a1b82538da435ece41c7279b7e48e52304953c9fa","Warning":""}
______________ create container
{"Id":"911faadcd28e2c581299a0c42182edb92d1ba3e0fb817ba3f110265782481758","Warnings":[]}
______________ start container
______________ wait for container
{"StatusCode":0,"Error":{"Message":"no container with ID 911faadcd28e2c581299a0c42182edb92d1ba3e0fb817ba3f110265782481758 found in database: no such container"}}
______________ output from container command:
Trying "kernel.org"
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 56842
;; flags: qr rd ra; QUERY: 1, ANSWER: 7, AUTHORITY: 0, ADDITIONAL: 0

;; QUESTION SECTION:
;kernel.org.            IN  ANY

;; ANSWER SECTION:
kernel.org.     85770   IN  SOA ns11.constellix.com. dns.constellix.com. 2015011869 43200 3600 1209600 180
kernel.org.     1580    IN  NS  ns21.constellix.com.
kernel.org.     1580    IN  NS  ns41.constellix.net.
kernel.org.     1580    IN  NS  ns61.constellix.net.
kernel.org.     1580    IN  NS  ns51.constellix.net.
kernel.org.     1580    IN  NS  ns31.constellix.com.
kernel.org.     1580    IN  NS  ns11.constellix.com.

Received 210 bytes from 10.89.27.1#53 in 12 ms

Step 11 prints the following output:

______________ build image
{"stream":"Step 1/3 : FROM docker.io/library/fedora:40"}
{"stream":"\n"}
{"stream":" ---\u003e 70d6fb7e34f4\n"}
{"stream":"Step 2/3 : RUN dnf -y install bind-utils"}
{"stream":"\n"}
{"stream":" ---\u003e Using cache\n"}
{"stream":" ---\u003e 9ceb28025fe2\n"}
{"stream":"Step 3/3 : CMD sh -c \"host -a kernel.org \u003e /resultdir/result 2\u003e\u00261\""}
{"stream":"\n"}
{"stream":" ---\u003e Using cache\n"}
{"stream":" ---\u003e aaf792bbe57f\n"}
{"stream":"Successfully built aaf792bbe57f\n"}
{"stream":"Successfully tagged localhost/test3:latest\n"}
______________ create network
{"Id":"917001373a5f0162ecf1b0a377063a43430b70073b8160b258f68bb330cb0b0f","Warning":""}
______________ create container
{"Id":"8d893afb5a4e2d49c9ee19de897c13725e180382a2ad838aaa5cd014952f87df","Warnings":[]}
______________ start container
______________ wait for container
{"StatusCode":1}
______________ output from container command:
Trying "kernel.org"
Host kernel.org not found: 2(SERVFAIL)
Received 28 bytes from 127.0.0.11#53 in 1 ms

Describe the results you expected

I would have expected seeing

______________ output from container command:
Trying "kernel.org"
Host kernel.org not found: 2(SERVFAIL)

in step 4 and step 10, in a similar way as it can be seen in step 11.

podman info output

host:
  arch: arm64
  buildahVersion: 1.35.4
  cgroupControllers:
  - cpu
  - memory
  - pids
  cgroupManager: systemd
  cgroupVersion: v2
  conmon:
    package: conmon-2.1.10-1.fc40.aarch64
    path: /usr/bin/conmon
    version: 'conmon version 2.1.10, commit: '
  cpuUtilization:
    idlePercent: 96.11
    systemPercent: 1.71
    userPercent: 2.18
  cpus: 2
  databaseBackend: sqlite
  distribution:
    distribution: fedora
    variant: coreos
    version: "40"
  eventLogger: journald
  freeLocks: 2048
  hostname: fcos-next5
  idMappings:
    gidmap:
    - container_id: 0
      host_id: 1002
      size: 1
    - container_id: 1
      host_id: 655360
      size: 65536
    uidmap:
    - container_id: 0
      host_id: 1002
      size: 1
    - container_id: 1
      host_id: 655360
      size: 65536
  kernel: 6.8.9-300.fc40.aarch64
  linkmode: dynamic
  logDriver: journald
  memFree: 412495872
  memTotal: 4082565120
  networkBackend: netavark
  networkBackendInfo:
    backend: netavark
    dns:
      package: aardvark-dns-1.10.0-1.fc40.aarch64
      path: /usr/libexec/podman/aardvark-dns
      version: aardvark-dns 1.10.0
    package: netavark-1.10.3-3.fc40.aarch64
    path: /usr/libexec/podman/netavark
    version: netavark 1.10.3
  ociRuntime:
    name: crun
    package: crun-1.15-1.fc40.aarch64
    path: /usr/bin/crun
    version: |-
      crun version 1.15
      commit: e6eacaf4034e84185fd8780ac9262bbf57082278
      rundir: /run/user/1002/crun
      spec: 1.0.0
      +SYSTEMD +SELINUX +APPARMOR +CAP +SECCOMP +EBPF +CRIU +LIBKRUN +WASM:wasmedge +YAJL
  os: linux
  pasta:
    executable: /usr/bin/pasta
    package: passt-0^20240510.g7288448-1.fc40.aarch64
    version: |
      pasta 0^20240510.g7288448-1.fc40.aarch64-pasta
      Copyright Red Hat
      GNU General Public License, version 2 or later
        <https://www.gnu.org/licenses/old-licenses/gpl-2.0.html>
      This is free software: you are free to change and redistribute it.
      There is NO WARRANTY, to the extent permitted by law.
  remoteSocket:
    exists: true
    path: /run/user/1002/podman/podman.sock
  security:
    apparmorEnabled: false
    capabilities: CAP_CHOWN,CAP_DAC_OVERRIDE,CAP_FOWNER,CAP_FSETID,CAP_KILL,CAP_NET_BIND_SERVICE,CAP_SETFCAP,CAP_SETGID,CAP_SETPCAP,CAP_SETUID,CAP_SYS_CHROOT
    rootless: true
    seccompEnabled: true
    seccompProfilePath: /usr/share/containers/seccomp.json
    selinuxEnabled: true
  serviceIsRemote: false
  slirp4netns:
    executable: /usr/bin/slirp4netns
    package: slirp4netns-1.2.2-2.fc40.aarch64
    version: |-
      slirp4netns version 1.2.2
      commit: 0ee2d87523e906518d34a6b423271e4826f71faf
      libslirp: 4.7.0
      SLIRP_CONFIG_VERSION_MAX: 4
      libseccomp: 2.5.3
  swapFree: 0
  swapTotal: 0
  uptime: 13h 6m 55.00s (Approximately 0.54 days)
  variant: v8
plugins:
  authorization: null
  log:
  - k8s-file
  - none
  - passthrough
  - journald
  network:
  - bridge
  - macvlan
  - ipvlan
  volume:
  - local
registries:
  search:
  - registry.fedoraproject.org
  - registry.access.redhat.com
  - docker.io
  - quay.io
store:
  configFile: /var/home/test12/.config/containers/storage.conf
  containerStore:
    number: 0
    paused: 0
    running: 0
    stopped: 0
  graphDriverName: overlay
  graphOptions: {}
  graphRoot: /var/home/test12/.local/share/containers/storage
  graphRootAllocated: 26238496768
  graphRootUsed: 6452252672
  graphStatus:
    Backing Filesystem: xfs
    Native Overlay Diff: "true"
    Supports d_type: "true"
    Supports shifting: "false"
    Supports volatile: "true"
    Using metacopy: "false"
  imageCopyTmpDir: /var/tmp
  imageStore:
    number: 6
  runRoot: /run/user/1002/containers
  transientStore: false
  volumePath: /var/home/test12/.local/share/containers/storage/volumes
version:
  APIVersion: 5.0.3
  Built: 1715299200
  BuiltTime: Fri May 10 00:00:00 2024
  GitCommit: ""
  GoVersion: go1.22.2
  Os: linux
  OsArch: linux/arm64
  Version: 5.0.3

Podman in a container

Privileged Or Rootless

Privileged

Upstream Latest Release

Yes

Additional environment details

$ docker --version
Docker version 26.1.2, build 211e74b

$ docker version
Client: Docker Engine - Community
 Version:           26.1.2
 API version:       1.45
 Go version:        go1.21.10
 Git commit:        211e74b
 Built:             Wed May  8 14:01:10 2024
 OS/Arch:           linux/arm64
 Context:           default

Server: Docker Engine - Community
 Engine:
  Version:          26.1.2
  API version:      1.45 (minimum version 1.24)
  Go version:       go1.21.10
  Git commit:       ef1912d
  Built:            Wed May  8 13:59:56 2024
  OS/Arch:          linux/arm64
  Experimental:     false
 containerd:
  Version:          1.6.31
  GitCommit:        e377cd56a71523140ca6ae87e30244719194a521
 runc:
  Version:          1.1.12
  GitCommit:        v1.1.12-0-g51d5e94
 docker-init:
  Version:          0.19.0
  GitCommit:        de40ad0

$ rpm -qf /usr/bin/docker
docker-ce-cli-26.1.2-1.fc40.aarch64

$ rpm -qa | grep docker
docker-compose-plugin-2.27.0-1.fc40.aarch64
docker-buildx-plugin-0.14.0-1.fc40.aarch64
docker-ce-cli-26.1.2-1.fc40.aarch64
docker-ce-rootless-extras-26.1.2-1.fc40.aarch64
docker-ce-26.1.2-1.fc40.aarch64

Additional information

Additional information like issue happens only occasionally or issue happens with a particular architecture or on a particular setting

containers / podman