domain policy not resolving the ip in nftset automatically

[crushed112]

Describe the bug

When defining a domain-based policy (dest_addr) in the PBR configuration, the expected behavior is for dnsmasq to dynamically resolve the domain and populate the associated nftset. However, the nftset remains empty, and the policy does not apply unless the IP is manually added to the nftset.

Your configs

1. /etc/config/dhcp

root@bpi-r3:~# cat /etc/config/dhcp
config dnsmasq
    option domainneeded '1'
    option localise_queries '1'
    option rebind_protection '1'
    option rebind_localhost '1'
    option local '/lan/'
    option domain 'lan'
    option expandhosts '1'
    option cachesize '1000'
    option authoritative '1'
    option readethers '1'
    option leasefile '/tmp/dhcp.leases'
    option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
    option localservice '1'
    option ednspacket_max '1232'
    option logqueries '1'
    option logfacility '/var/log/dnsmasq.log'

config dhcp 'lan'
    option interface 'lan'
    option start '100'
    option limit '150'
    option leasetime '12h'
    option dhcpv4 'server'
    option force '1'

config dhcp 'wan'
    option interface 'wan'
    option ignore '1'

config odhcpd 'odhcpd'
    option maindhcp '0'
    option leasefile '/tmp/hosts/odhcpd'
    option leasetrigger '/usr/sbin/odhcpd-update'
    option loglevel '4'

config host
    option name 'npi-r2s'
    list mac 'XX:XX:XX:XX:XX:XX'
    option ip '192.168.1.243'
    option leasetime 'infinite'

2. /etc/config/network

root@bpi-r3:~# cat /etc/config/dhcp

config dnsmasq
    option domainneeded '1'
    option localise_queries '1'
    option rebind_protection '1'
    option rebind_localhost '1'
    option local '/lan/'
    option domain 'lan'
    option expandhosts '1'
    option cachesize '1000'
    option authoritative '1'
    option readethers '1'
    option leasefile '/tmp/dhcp.leases'
    option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
    option localservice '1'
    option ednspacket_max '1232'
    option logqueries '1'
    option logfacility '/var/log/dnsmasq.log'

config dhcp 'lan'
    option interface 'lan'
    option start '100'
    option limit '150'
    option leasetime '12h'
    option dhcpv4 'server'
    option force '1'

config dhcp 'wan'
    option interface 'wan'
    option ignore '1'

config odhcpd 'odhcpd'
    option maindhcp '0'
    option leasefile '/tmp/hosts/odhcpd'
    option leasetrigger '/usr/sbin/odhcpd-update'
    option loglevel '4'

config host
    option name 'npi-r2s'
    list mac 'XX:XX:XX:XX:XX:XX'
    option ip '192.168.1.243'
    option leasetime 'infinite'

root@bpi-r3:~# cat /etc/config/network

config interface 'loopback'
    option device 'lo'
    option proto 'static'
    option ipaddr '127.0.0.1'
    option netmask '255.0.0.0'

config globals 'globals'
    option ula_prefix 'fdxx:xxxx:xxxx::/48'

config device
    option name 'br-lan'
    option type 'bridge'
    list ports 'lan1'
    list ports 'lan2'
    list ports 'lan3'
    list ports 'lan4'
    list ports 'sfp2'

config interface 'lan'
    option device 'br-lan'
    option proto 'static'
    option ipaddr '192.168.1.1'
    option netmask '255.255.255.0'
    list dns '192.168.1.243'

config device
    option name 'br-wan'
    option type 'bridge'
    list ports 'eth1'
    list ports 'wan'

config device
    option name 'eth1'
    option macaddr 'XX:XX:XX:XX:XX:XX'

config device
    option name 'wan'
    option macaddr 'XX:XX:XX:XX:XX:XX'

config interface 'wan'
    option device 'br-wan'
    option proto 'dhcp'
    option peerdns '0'
    list dns '192.168.1.243'

config interface 'vpnwan_CO_136'
    option proto 'wireguard'
    option private_key '...'
    list addresses '10.2.0.2/32'
    option nohostroute '1'
    option defaultroute '0'

config wireguard_vpnwan_CO_136
    option description 'Imported peer configuration'
    option public_key '...'
    list allowed_ips '0.0.0.0/0'
    option endpoint_host 'xxx.xxx.xxx.xxx'
    option endpoint_port 'xxxxx'

root@bpi-r3:~# 

3. /etc/config/firewall

root@bpi-r3:~# cat /etc/config/firewall

config defaults
    option syn_flood '1'
    option input 'REJECT'
    option output 'ACCEPT'
    option forward 'REJECT'

config zone
    option name 'lan'
    option input 'ACCEPT'
    option output 'ACCEPT'
    option forward 'ACCEPT'
    list network 'lan'

config zone
    option name 'wan'
    option input 'REJECT'
    option output 'ACCEPT'
    option forward 'DROP'
    option masq '1'
    option mtu_fix '1'
    list network 'wan'
    list network 'vpnwan_CO_136'

config forwarding
    option src 'lan'
    option dest 'wan'

config rule
    option name 'Allow-DHCP-Renew'
    option src 'wan'
    option proto 'udp'
    option dest_port '68'
    option target 'ACCEPT'
    option family 'ipv4'

config rule
    option name 'Allow-Ping'
    option src 'wan'
    option proto 'icmp'
    option icmp_type 'echo-request'
    option family 'ipv4'
    option target 'ACCEPT'

config rule
    option name 'Allow-IGMP'
    option src 'wan'
    option proto 'igmp'
    option family 'ipv4'
    option target 'ACCEPT'

config rule
    option name 'Allow-DHCPv6'
    option src 'wan'
    option proto 'udp'
    option dest_port '546'
    option family 'ipv6'
    option target 'ACCEPT'

config rule
    option name 'Allow-MLD'
    option src 'wan'
    option proto 'icmp'
    option src_ip 'fe80::/10'
    list icmp_type '130/0'
    list icmp_type '131/0'
    list icmp_type '132/0'
    list icmp_type '143/0'
    option family 'ipv6'
    option target 'ACCEPT'

config rule
    option name 'Allow-ICMPv6-Input'
    option src 'wan'
    option proto 'icmp'
    list icmp_type 'echo-request'
    list icmp_type 'echo-reply'
    list icmp_type 'destination-unreachable'
    list icmp_type 'packet-too-big'
    list icmp_type 'time-exceeded'
    list icmp_type 'bad-header'
    list icmp_type 'unknown-header-type'
    list icmp_type 'router-solicitation'
    list icmp_type 'neighbour-solicitation'
    list icmp_type 'router-advertisement'
    list icmp_type 'neighbour-advertisement'
    option limit '1000/sec'
    option family 'ipv6'
    option target 'ACCEPT'

config rule
    option name 'Allow-ICMPv6-Forward'
    option src 'wan'
    option dest '*'
    option proto 'icmp'
    list icmp_type 'echo-request'
    list icmp_type 'echo-reply'
    list icmp_type 'destination-unreachable'
    list icmp_type 'packet-too-big'
    list icmp_type 'time-exceeded'
    list icmp_type 'bad-header'
    list icmp_type 'unknown-header-type'
    option limit '1000/sec'
    option family 'ipv6'
    option target 'ACCEPT'

config rule
    option name 'Allow-IPSec-ESP'
    option src 'wan'
    option dest 'lan'
    option proto 'esp'
    option target 'ACCEPT'

config rule
    option name 'Allow-ISAKMP'
    option src 'wan'
    option dest 'lan'
    option dest_port '500'
    option proto 'udp'
    option target 'ACCEPT'

config include 'pbr'
    option fw4_compatible '1'
    option type 'script'
    option path '/usr/share/pbr/firewall.include'

4. /etc/config/pbr

root@bpi-r3:~# cat /etc/config/pbr

config pbr 'config'
    option enabled '1'
    option verbosity '2'
    option strict_enforcement '1'
    option resolver_set 'dnsmasq.nftset'
    list resolver_instance '*'
    option ipv6_enabled '1'
    option boot_timeout '30'
    option rule_create_option 'add'
    option procd_boot_delay '0'
    option procd_reload_delay '1'
    option webui_show_ignore_target '1'
    option nft_rule_counter '1'
    option nft_set_auto_merge '1'
    option nft_set_counter '1'
    option nft_set_flags_interval '1'
    option nft_set_flags_timeout '0'
    option nft_set_policy 'performance'
    list webui_supported_protocol 'all'

config policy
    option name 'testvpnpolicy'
    option src_addr '192.168.1.0/24'
    option dest_addr 'dnsleaktest.com'
    option interface 'vpnwan_CO_136'

config policy
    option name 'vpn'
    option interface 'vpnwan_CO_136'
    option src_addr '192.168.1.0/24'
    option enabled '0'

To Reproduce

I admit I haven't done a clean reproduction of this so this could be due to my environment, ( bpi-r3 on snapshot, dnsmasq-full and luci-app-pbr installed via apk, dnsmasq forwarding requests to a local dns server on same subnet as my bpi-r3, proton vpn wireguard interface)

Steps to reproduce the behavior:

1. add a policy with a domain for dest_addr
2. See error

Expected behavior

I expect

root@bpi-r3:~# nft list set inet fw4 pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5

to return

table inet fw4 {
    set pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5 {
        type ipv4_addr
        flags interval
        counter
        auto-merge
        comment "testvpnpolicy"
        elements = { xx.xxx.xx.xxx ... }
    }
}

but I get

table inet fw4 {
    set pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5 {
        type ipv4_addr
        flags interval
        counter
        auto-merge
        comment "testvpnpolicy"
    }
}

Policy Routing run-time information

1. Output of /etc/init.d/pbr reload with verbosity set to 2:

root@bpi-r3:~# /etc/init.d/pbr reload
Loading environment (on_start) Using wan interface (on_start): wan 
Found wan gateway (on_start): xxx.xxx.xxx.xxx 
Using wan6 interface (on_start): wan6 
Setting up routing for 'wan/br-wan/xxx.xxx.xxx.xxx/::/0' [✓]
Setting up routing for 'vpnwan_CO_136/10.2.0.2/::/0' [✓]
Routing 'testvpnpolicy' via vpnwan_CO_136 [✓]
Installing fw4 nft file [✓]
pbr 1.1.7-47 monitoring interfaces: wan vpnwan_CO_136 
pbr 1.1.7-47 (fw4 nft file mode) started with gateways:
wan/br-wan/xxx.xxx.xxx.xxx/::/0 [✓]
vpnwan_CO_136/10.2.0.2/::/0
root@bpi-r3:~# 

2. Output of /etc/init.d/pbr status:

root@bpi-r3:~# /etc/init.d/pbr status

pbr - environment
pbr 1.1.7-47 running on OpenWrt SNAPSHOT.

Dnsmasq version 2.90  Copyright (c) 2000-2024 Simon Kelley
Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack no-ipset nftset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile

pbr fw4 nft file: /usr/share/nftables.d/ruleset-post/30-pbr.nft
add chain inet fw4 pbr_mark_0x010000
add rule inet fw4 pbr_mark_0x010000 counter mark set mark and 0xff00ffff xor 0x010000
add rule inet fw4 pbr_mark_0x010000 return
add chain inet fw4 pbr_mark_0x020000
add rule inet fw4 pbr_mark_0x020000 counter mark set mark and 0xff00ffff xor 0x020000
add rule inet fw4 pbr_mark_0x020000 return
add set inet fw4 pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5 { type ipv4_addr;          auto-merge;         counter;        flags interval;             policy performance;                 comment "testvpnpolicy";}
add set inet fw4 pbr_vpnwan_CO_136_6_dst_ip_cfg026ff5 { type ipv6_addr;          auto-merge;         counter;        flags interval;             policy performance;                 comment "testvpnpolicy";}
add rule inet fw4 pbr_prerouting ip saddr { 192.168.1.0/24 } ip daddr @pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5 counter goto pbr_mark_0x020000 comment "testvpnpolicy"

pbr chains - policies
    chain pbr_forward { # handle 30
    }
    chain pbr_input { # handle 31
    }
    chain pbr_output { # handle 32
    }
    chain pbr_postrouting { # handle 34
    }
    chain pbr_prerouting { # handle 33
        ip saddr 192.168.1.0/24 ip daddr @pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5 counter packets 0 bytes 0 goto pbr_mark_0x020000 comment "testvpnpolicy" # handle 1643
    }
    chain pbr_dstnat { # handle 29
    }

pbr chains - marking
    chain pbr_mark_0x010000 { # handle 1635
        counter packets 0 bytes 0 meta mark set meta mark & 0xff01ffff | 0x00010000 # handle 1636
        return # handle 1637
    }
    chain pbr_mark_0x020000 { # handle 1638
        counter packets 0 bytes 0 meta mark set meta mark & 0xff02ffff | 0x00020000 # handle 1639
        return # handle 1640
    }

pbr nft sets
    set pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5 { # handle 1641
        type ipv4_addr
        flags interval
        counter
        auto-merge
        comment "testvpnpolicy"
    }
    set pbr_vpnwan_CO_136_6_dst_ip_cfg026ff5 { # handle 1642
        type ipv6_addr
        flags interval
        counter
        auto-merge
        comment "testvpnpolicy"
    }

dnsmasq sets
nftset=/dnsleaktest.com/4#inet#fw4#pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5,6#inet#fw4#pbr_vpnwan_CO_136_6_dst_ip_cfg026ff5 # testvpnpolicy

IPv4 table 256 route: default via xxx.xxx.xxx.xxx dev br-wan 
IPv4 table 256 rule(s):
30000:  from all fwmark 0x10000/0xff0000 lookup pbr_wan
IPv4 table 257 route: default via 10.2.0.2 dev vpnwan_CO_136 
IPv4 table 257 rule(s):
29998:  from all fwmark 0x20000/0xff0000 lookup pbr_vpnwan_CO_136

root@bpi-r3:~# ubus call system board
{
    "kernel": "6.6.61",
    "hostname": "bpi-r3",
    "system": "ARMv8 Processor rev 4",
    "model": "Bananapi BPI-R3",
    "board_name": "bananapi,bpi-r3",
    "rootfs_type": "squashfs",
    "release": {
        "distribution": "OpenWrt",
        "version": "SNAPSHOT",
        "revision": "r28146-52b6c92479",
        "target": "mediatek/filogic",
        "description": "OpenWrt SNAPSHOT r28146-52b6c92479",
        "builddate": "1732223081"
    }
}

[crushed112]

running

nft add element inet fw4 pbr_vpnwan_CO_136_4_dst_ip_cfg026ff5 { 23.239.16.110 }

does make the policy work also

[stangri]

Hey, thanks for the donation and Happy Thanksgiving!

How much of the https://docs.openwrt.melmac.net/pbr/#AWordAboutBrokenDomainPolicies have you gone thru? For majority of people the last big bullet point in that section (then further divided in sub-bullet points) applies.

From your configs I'd venture a guess that you're using a pihole (or another resolver) on your LAN, so it is likely that your clients do not make any resolution requests to your router/dnsmasq.

[crushed112]

Happy thanksgiving and thanks for the fast response!

Ah I didn't actually read that full section! I saw the first bit and misread it as the router just needs to be able to make DNS requests which I assumed was just to resolve the IP

I didn't realize the router actually needs to be the DNS server

Yes you're right I do have Adguard home separate from my router, does this mean I won't be able to do domain based policies?

Thank you, I guess I didn't make the connection that the policies would actually be enforced when the domains were resolved

[stangri]

Yes you're right I do have Adguard home separate from my router, does this mean I won't be able to do domain based policies?

You have a few options on how to make pbr work.

1. Instead of configuring your local devices to use AGH as the resolver, you set the dnsmasq to use AGH as the resolver and let local devices use your router/dnsmasq for resolution. That would make pbr work, but the reports in AGH will show one source of DNS requests -- your router.
2. If you have a router (like Dell Edge 620, which can be bought second hand on ebay for under $100), you can run AGH on your router. AGH can populate ipsets and that option is supported in the (pre-nft) iptables/ipset-flavour of pbr, so there's a hope that in the future AGH versions would also learn to populate nft sets and it can then be used with the regular version of pbr. That way you can keep using AGH for the domain resolution and it would populate nft sets that pbr creates (again, once AGH is nft-aware).

[crushed112]

Ah option 1 sounds ideal, I could run it on this router too but I'm going to try option 1 first thanks so much I'll close this issue as complete now