isc-dhcp-server-ipv6: DDNS updates lead to segmentation fault

jnhmn commented 4 years ago

Maintainer: @pprindeville Environment: ar71xx-generic-archer-c7-v2 (v19.07.2)

Description: While trying to send DDNS Updates to an ISC Bind9 Server (9.10.3-P4-Debian) the DHCP Server terminates with an segfault. The same config file works with an Openwrt 18.06.2

/etc/dhcpd.conf:

ignore client-updates;
update-static-leases on;

default-lease-time 86400;
max-lease-time 604800;
ddns-update-style interim;

option domain-name-servers 10.214.4.1;

subnet 10.214.4.0 netmask 255.255.255.0 {
  range 10.214.4.50 10.214.4.100;
  option routers 10.214.4.1;
  option domain-name-servers 10.214.4.1,10.214.0.1;
  option domain-name "lan.example.com";
  option domain-search "lan.example.com";
  ddns-domainname "lan.example.com";
}

include "/root/ddns.key";

zone lan.example.com. {
  primary 10.214.0.1;
  key lan.example.com.;
}

/root/ddns.key:

{
    algorithm HMAC-MD5;
    secret "---confidential---";
};

Diffconfig:

CONFIG_TARGET_ar71xx_generic=y
CONFIG_TARGET_ar71xx_generic_DEVICE_archer-c7-v2=y
CONFIG_DEVEL=y
CONFIG_TOOLCHAINOPTS=y
CONFIG_ATH10K_LEDS=y
CONFIG_ATH10K_THERMAL=y
CONFIG_BUILD_PATENTED=y
# CONFIG_KERNEL_CRASHLOG is not set
# CONFIG_KERNEL_DEBUG_INFO is not set
# CONFIG_KERNEL_DEBUG_KERNEL is not set
# CONFIG_KERNEL_ELF_CORE is not set
# CONFIG_KERNEL_KALLSYMS is not set
# CONFIG_KERNEL_MAGIC_SYSRQ is not set
CONFIG_OPENSSL_ENGINE=y
CONFIG_OPENSSL_PREFER_CHACHA_OVER_GCM=y
CONFIG_OPENSSL_WITH_ASM=y
CONFIG_OPENSSL_WITH_CHACHA_POLY1305=y
CONFIG_OPENSSL_WITH_CMS=y
CONFIG_OPENSSL_WITH_DEPRECATED=y
CONFIG_OPENSSL_WITH_ERROR_MESSAGES=y
CONFIG_OPENSSL_WITH_NPN=y
CONFIG_OPENSSL_WITH_PSK=y
CONFIG_OPENSSL_WITH_SRP=y
CONFIG_OPENSSL_WITH_TLS13=y
CONFIG_OPENVPN_openssl_ENABLE_DEF_AUTH=y
CONFIG_OPENVPN_openssl_ENABLE_FRAGMENT=y
CONFIG_OPENVPN_openssl_ENABLE_LZ4=y
CONFIG_OPENVPN_openssl_ENABLE_LZO=y
CONFIG_OPENVPN_openssl_ENABLE_MULTIHOME=y
CONFIG_OPENVPN_openssl_ENABLE_PF=y
CONFIG_OPENVPN_openssl_ENABLE_PORT_SHARE=y
CONFIG_OPENVPN_openssl_ENABLE_SERVER=y
CONFIG_OPENVPN_openssl_ENABLE_SMALL=y
CONFIG_PACKAGE_6in4=y
CONFIG_PACKAGE_6to4=y
CONFIG_PACKAGE_ath10k-firmware-qca988x=y
# CONFIG_PACKAGE_ath10k-firmware-qca988x-ct is not set
CONFIG_PACKAGE_bind-libs=y
CONFIG_PACKAGE_bind-server=y
CONFIG_PACKAGE_bind-server-filter-aaaa=y
CONFIG_PACKAGE_bird2=y
CONFIG_PACKAGE_block-mount=y
CONFIG_PACKAGE_bsdtar=y
CONFIG_PACKAGE_cgi-io=y
# CONFIG_PACKAGE_dnsmasq is not set
CONFIG_PACKAGE_e2fsprogs=y
CONFIG_PACKAGE_gdb=y
CONFIG_PACKAGE_gdbserver=y
CONFIG_PACKAGE_gzip=y
CONFIG_PACKAGE_htop=y
CONFIG_PACKAGE_isc-dhcp-server-ipv6=y
CONFIG_PACKAGE_kmod-ath10k=y
# CONFIG_PACKAGE_kmod-ath10k-ct is not set
CONFIG_PACKAGE_kmod-crypto-aead=y
CONFIG_PACKAGE_kmod-crypto-crc32=y
CONFIG_PACKAGE_kmod-crypto-crc32c=y
CONFIG_PACKAGE_kmod-crypto-hash=y
CONFIG_PACKAGE_kmod-crypto-hmac=y
CONFIG_PACKAGE_kmod-crypto-manager=y
CONFIG_PACKAGE_kmod-crypto-md5=y
CONFIG_PACKAGE_kmod-crypto-null=y
CONFIG_PACKAGE_kmod-crypto-pcompress=y
CONFIG_PACKAGE_kmod-crypto-sha1=y
CONFIG_PACKAGE_kmod-crypto-sha256=y
CONFIG_PACKAGE_kmod-crypto-sha512=y
CONFIG_PACKAGE_kmod-fs-exfat=y
CONFIG_PACKAGE_kmod-fs-ext4=y
CONFIG_PACKAGE_kmod-fs-f2fs=y
CONFIG_PACKAGE_kmod-fs-ntfs=y
CONFIG_PACKAGE_kmod-fs-vfat=y
CONFIG_PACKAGE_kmod-ip6tables-extra=y
CONFIG_PACKAGE_kmod-ipt-iprange=y
CONFIG_PACKAGE_kmod-iptunnel=y
CONFIG_PACKAGE_kmod-iptunnel4=y
CONFIG_PACKAGE_kmod-lib-crc16=y
CONFIG_PACKAGE_kmod-lib-crc32c=y
CONFIG_PACKAGE_kmod-nls-cp437=y
CONFIG_PACKAGE_kmod-nls-iso8859-1=y
CONFIG_PACKAGE_kmod-nls-utf8=y
CONFIG_PACKAGE_kmod-scsi-core=y
CONFIG_PACKAGE_kmod-sit=y
CONFIG_PACKAGE_kmod-thermal=y
CONFIG_PACKAGE_kmod-tun=y
CONFIG_PACKAGE_kmod-usb-storage=y
CONFIG_PACKAGE_libarchive=y
CONFIG_PACKAGE_libatomic=y
CONFIG_PACKAGE_libbfd=y
CONFIG_PACKAGE_libblkid=y
CONFIG_PACKAGE_libbotan=y
CONFIG_PACKAGE_libbz2=y
CONFIG_PACKAGE_libcap=y
CONFIG_PACKAGE_libcomerr=y
CONFIG_PACKAGE_libevent2-core=y
CONFIG_PACKAGE_libevent2-pthreads=y
CONFIG_PACKAGE_libexpat=y
CONFIG_PACKAGE_libext2fs=y
CONFIG_PACKAGE_libgd=y
CONFIG_PACKAGE_libiwinfo-lua=y
CONFIG_PACKAGE_libjpeg=y
CONFIG_PACKAGE_liblua=y
CONFIG_PACKAGE_liblucihttp=y
CONFIG_PACKAGE_liblucihttp-lua=y
CONFIG_PACKAGE_liblzma=y
CONFIG_PACKAGE_liblzo=y
CONFIG_PACKAGE_libmbedtls=y
CONFIG_PACKAGE_libncurses=y
CONFIG_PACKAGE_libopcodes=y
CONFIG_PACKAGE_libopenssl=y
CONFIG_PACKAGE_libopenssl-conf=y
CONFIG_PACKAGE_libpcap=y
CONFIG_PACKAGE_libpng=y
CONFIG_PACKAGE_libpopt=y
CONFIG_PACKAGE_libreadline=y
CONFIG_PACKAGE_librt=y
CONFIG_PACKAGE_libss=y
CONFIG_PACKAGE_libstdcpp=y
CONFIG_PACKAGE_libubus-lua=y
CONFIG_PACKAGE_libuci-lua=y
CONFIG_PACKAGE_libustream-openssl=y
CONFIG_PACKAGE_libuuid=y
CONFIG_PACKAGE_libzstd=y
CONFIG_PACKAGE_lua=y
CONFIG_PACKAGE_luasocket=y
CONFIG_PACKAGE_luci=y
CONFIG_PACKAGE_luci-app-firewall=y
CONFIG_PACKAGE_luci-app-ntpc=y
CONFIG_PACKAGE_luci-app-openvpn=y
CONFIG_PACKAGE_luci-app-opkg=y
CONFIG_PACKAGE_luci-app-vnstat=y
CONFIG_PACKAGE_luci-base=y
CONFIG_PACKAGE_luci-compat=y
CONFIG_PACKAGE_luci-lib-ip=y
CONFIG_PACKAGE_luci-lib-jsonc=y
CONFIG_PACKAGE_luci-lib-nixio=y
CONFIG_PACKAGE_luci-mod-admin-full=y
CONFIG_PACKAGE_luci-mod-network=y
CONFIG_PACKAGE_luci-mod-status=y
CONFIG_PACKAGE_luci-mod-system=y
CONFIG_PACKAGE_luci-proto-ipv6=y
CONFIG_PACKAGE_luci-proto-ppp=y
CONFIG_PACKAGE_luci-ssl-openssl=y
CONFIG_PACKAGE_luci-theme-bootstrap=y
CONFIG_PACKAGE_ntp-utils=y
CONFIG_PACKAGE_ntpclient=y
CONFIG_PACKAGE_ntpd=y
CONFIG_PACKAGE_openssl-util=y
CONFIG_PACKAGE_openvpn-openssl=y
CONFIG_PACKAGE_prometheus-node-exporter-lua=y
CONFIG_PACKAGE_prometheus-node-exporter-lua-nat_traffic=y
CONFIG_PACKAGE_prometheus-node-exporter-lua-netstat=y
CONFIG_PACKAGE_prometheus-node-exporter-lua-openwrt=y
CONFIG_PACKAGE_prometheus-node-exporter-lua-wifi=y
CONFIG_PACKAGE_px5g-standalone=y
CONFIG_PACKAGE_rp-pppoe-common=y
CONFIG_PACKAGE_rpcd=y
CONFIG_PACKAGE_rpcd-mod-file=y
CONFIG_PACKAGE_rpcd-mod-iwinfo=y
CONFIG_PACKAGE_rpcd-mod-luci=y
CONFIG_PACKAGE_rpcd-mod-rrdns=y
CONFIG_PACKAGE_rsync=y
CONFIG_PACKAGE_tcpdump-mini=y
CONFIG_PACKAGE_terminfo=y
CONFIG_PACKAGE_tinc=y
CONFIG_PACKAGE_uhttpd=y
CONFIG_PACKAGE_uhttpd-mod-ubus=y
CONFIG_PACKAGE_vim-full=y
CONFIG_PACKAGE_vnstat=y
CONFIG_PACKAGE_vnstati=y
CONFIG_PACKAGE_wireless-tools=y
CONFIG_PACKAGE_wpad=y
CONFIG_PACKAGE_wpad-basic=m
CONFIG_PACKAGE_xz-utils=y
CONFIG_PACKAGE_zlib=y
CONFIG_PACKAGE_zoneinfo-europe=y
CONFIG_PACKAGE_zstd=y
CONFIG_RSYNC_zlib=y
CONFIG_USE_LIBSTDCXX=y
CONFIG_USE_RFKILL=y
# CONFIG_USE_UCLIBCXX is not set
CONFIG_ZSTD_OPTIMIZE_O3=y

Stacktrace:

#1  0x0058ea31 in isc_msgcat_get (msgcat=<optimized out>, set=<optimized out>, message=<optimized out>, 
    default_text=<optimized out>) at msgcat.c:118
#2  0x00576a53 in isc_result_tomany_helper (result=20, tables=<optimized out>) at result.c:267
#3  0x00576adf in isc_result_totext (result=20) at result.c:283
#4  0x00436e87 in ddns_fwd_srv_add1 (ddns_cb=0x6a3a50, eresult=20) at ddns.c:1571
#5  0x0044a797 in ddns_interlude (taskp=<optimized out>, eventp=<optimized out>) at dns.c:2477
#6  0x0057d28b in dispatch (manager=0x77eaf010) at task.c:1140
#7  isc__taskmgr_dispatch (manager0=<optimized out>) at task.c:1653
#8  0x005803d7 in evloop (ctx=0x77eae010) at app.c:508
#9  0x005807ad in isc__app_ctxrun (ctx0=0x77eae010) at app.c:624
#10 0x0058105d in isc_app_ctxrun (ctx=<optimized out>) at ../app_api.c:95
#11 0x00448c7f in dispatch () at dispatch.c:114
#12 0x0041b461 in main (argc=<optimized out>, argv=<optimized out>) at dhcpd.c:1064

I can attach a gdb session to the affected daemon, if you need further details. But I have no debugging experience.

moocan commented 4 years ago

strace_isc-dhcp-server-ipv6.log Hello,

I have too a Segmentation fault on a Netgear R7800 with isc-dhcp-server-ipv6 on OpenWRT 19.07.2.

You can find included output of strace. How can I help ?

Kind Regards

DarwinSurvivor commented 4 years ago

I'm getting exactly the same problem in an Alpine docker instance, so this may be an upstream problem.

Setting ddns-update-style to none stops it from crashing, though obviously that means the DNS server isn't getting updated.

pprindeville commented 4 years ago

I don't have time to dig deep on this but if someone finds a patch upstream I'll either refresh the package to a new version containing the fix or backport it if it's not yet released...

moocan commented 4 years ago

Hello,

Many thanks for your return.

Before reverting my device back from 19.07.2 to 18.06.2 and loosing all I updated isc-dhcp-* 4.4.1 packages with some isc patches and then to the last 4.4.2 version and using ever the same configuration ... segmentation fault

Then I have reverted back the R7800 device to 18.06.2. I have recreated all interfaces, VLAN, routing rules ... and other configuration on this device taking into account that R7800 is less supported on 18.06.2 (many updates for this devices with 19.07.2 as for other devices).

Using bind and isc-dhcp-server-ipv6 18.06.2 stock version (Bind 9.11.14-1 / isc-dhcp-server-ipv6 4.4.1-3) and exactly and same bind configuration, zone file and dhcpd configuration ... no segmentation fault.

Then I have I have backported bind 19.07.2 stock version (Bind 9.14.8-1 / isc-dhcp-server-ipv6 4.4.1-3) to 18.06.2 using buildsystem and with exactly the same bind configuration, zone file and dhcpd configuration ... no segmentation fault. Then from 4.4.1 + patches to 4.4.2 ... no segmentation fault

Then using a proxmox server and a light ubuntu 18.04 server VM with Bind and dhcpd with same amount of RAM, interfaces, VLAN and using same conf files for bind to dhcpd ... no segmentation fault

If I am not discouraged and have time, I would do all these tests again with the latest 18.06 release thus 18.06.8

I'm not a developer, how can I diagnose this and report this to upstream or someone else ?

Thanks you in advance for your return.

PS: I would like to apologize for my English language mistakes. Kind Regards

pprindeville commented 4 years ago

It sounds like the issue might be architecture dependent, so I'm not surprised that you can't reproduce it on an x86-based VM.

Can you bisect it on the R7800 itself by trying different package versions (on master or 19.07.02) until you find the point that the regression crept in?

Thanks

pprindeville commented 4 years ago

You also might be able to run Qemu/KVM on a Raspberry Pi4 and emulate ARM execution.

Buddy-Matt commented 4 years ago

Hi,

I am also getting the same issue with an Archer C7 v2 running 19.07.3, r11063-85e04e9f46 - both dhcp and bind installed locally. Have tried both the ipv4 and ipv6 packages.

The apparent sequence of events appears to be: Start DHCP DHCPDISCOVER received DHCPOFFER sent DHCPREQUEST received DHCPACK sent Segfault

On the bind side of things, the A & TXT records are created in my zone file, the .jnl is created/updated, but nothing happens with the reverse zone (which is also configured as per pre-update)

Buddy-Matt commented 4 years ago

Looks like it may be the same as this: https://gitlab.alpinelinux.org/alpine/aports/issues/10934 with the equivalent of this bind patch fixing it: https://gitlab.isc.org/isc-projects/bind9/-/commit/daade37977fafee12c7b3c1483516e010d2b74a6

If I can work out how to build a version & install with this patch myself I'll let you know if it works.

Buddy-Matt commented 4 years ago

Can confirm applying the code edit from my second link to openwrt/build_dir/target-mips_24kc_musl/isc-dhcp-ipv6/dhcp-4.4.1/bind/bind-9.11.2-P1/lib/isc/nls/msgcat.c and re-compiling dhcpd (actually, recompiling nls, then lib, then bind-9.11.2-P1, then bind, then dhcp-4.4.1 for exact reproduction steps) fixes the issue.

This method did increase my dhcpd binary's size from 1.9Mb to 2.6Mb which I found odd... but it works now at least.

webgeek1234 commented 4 years ago

I updated a TP-Link Archer C2600 from an old snapshot to stable 19.07.3 and this issue appeared, using isc-dhcp-server-ipv4.

Tried to compile the existing package with the above linked patch using the build systems patch folder, but can't figure out how to get it to apply with the multiple layers of archives. Also tried compiling 4.4.2, since it was released a few days after said patch was committed, but that also ended in quick failure due to missing includes.

For the moment, I've disabled dynamic dns on my network, just to keep it from being completely offline, but hope to see this fixed soon to get features back to par.

pprindeville commented 4 years ago

Looking at how the NLS is built, it's from a tarball taken from tar that's embedded in the dhcp tarball itself. So it can't easily be patched because exploding bind/bind.tar.gz until after patches have been applied.

Further, looking at the dhcp-4.4.2-RELNOTES file, it doesn't look like this (2020/1/13) was submitted in time for the 4.4.2 release (2020/1/22).

mannimammut76 commented 3 years ago

heho... my problem was not to apply patches, i have done weirder things with the build system :-) it is my first post on github ever, as i do not like toooooo much accounts... but as others are facing this problem. here are my patches, put them in the patches folder...

--- /dev/null +++ b/bind/010-ddns.patch @@ -0,0 +1,27 @@ +diff --git a/lib/isc/nls/msgcat.c b/lib/isc/nls/msgcat.c +index ab09b9457942ff359af6cdc5babbac53d13e53a0..dd3d177fb3968c2f16e18ac8dca226f5ed8667a4 100644 +--- a/lib/isc/nls/msgcat.c ++++ b/lib/isc/nls/msgcat.c +@@ -62,9 +62,8 @@ isc_msgcat_open(const char *name, isc_msgcat_t **msgcatp) { +

ifdef HAVE_CATGETS
/ +- We don't check if catopen() fails because we don't care. +- If it does fail, then when we call catgets(), it will use +- the default string. ++ We don't check if catopen() fails because isc_msgcat_get() takes ++ care of that before calling catgets().
*/
msgcat->catalog = catopen(name, 0);
endif

+@@ -112,8 +111,9 @@ isc_msgcat_get(isc_msgcat_t *msgcat, int set, int message,
REQUIRE(default_text != NULL);
ifdef HAVE_CATGETS

+- if (msgcat == NULL) ++ if (msgcat == NULL || msgcat->catalog == (nl_catd)(-1)) {
return (default_text); ++ }
return (catgets(msgcat->catalog, set, message, default_text));
else
return (default_text);

--- a/bind/Makefile.in +++ b/bind/Makefile.in @@ -42,6 +42,9 @@ bind1: echo ${bindsrcdir} already unpacked... ; \ else \ gunzip -c bind.tar.gz | tar xf - ; \

cp 010-ddns.patch ${bindsrcdir}/ ; \
(cd ${bindsrcdir} ; \
patch -p1 < 010-ddns.patch) ; \ fi

Configure the libraries

mannimammut76 commented 3 years ago

sorry if not useful ... m first post on github ;-)

moocan commented 3 years ago

@mannimammut76 Hello, May I ask you to upload all your patches files on github (in your message) please ? You can attach your files just with a drag and drop at the bottom of the message field (after the dotted line). And where to put them .. if patches folder is feeds/packages/net/isc-dhcp/patches ? Sorry i'm not an expert of the build system :( It would be very nice.

Thank you in advance

Kind Regards

chriskreuzberger commented 3 years ago

Hi everybody, What is the current status of this issue? I have encountered the same problem that with 19.07, a DDNS update causes a segmentation fault. Is it possible to provide a package update to an existing release?

Many thanks for your help & kind regards!

pprindeville commented 3 years ago

This might have been fixed by PR #14196. Please retest.

webgeek1234 commented 3 years ago

I picked the above mentioned change back to 19.07 and built the ipv4 server package, then pushed it to a tp-link archer c2600. It is working with ddns-update-style set to standard. Can that change be officially backported to stable?

pprindeville commented 3 years ago

Sorry, which PR's are you asking to be cherry-picked where?

webgeek1234 commented 3 years ago

https://github.com/openwrt/packages/pull/14196 to 19.07. To fix the crash on the current stable releases.

pprindeville commented 3 years ago

14196 to 19.07. To fix the crash on the current stable releases.

@webgeek1234 See PR #14715.

webgeek1234 commented 3 years ago

Version 4.4.1-4, which is now available via opkg, does fix the crash. Which resolves this issue for me.

jnhmn commented 3 years ago

Works for me, too.

I would suggest to close this issue

Thank you very much :)

openwrt / packages

isc-dhcp-server-ipv6: DDNS updates lead to segmentation fault #11829

ifdef HAVE_CATGETS

endif

ifdef HAVE_CATGETS

else

Configure the libraries

14196 to 19.07. To fix the crash on the current stable releases.