search

stangri / source.openwrt.melmac.net

OpenWrt Packages
GNU General Public License v3.0
144 stars 47 forks source link

[policy-based-routing] Issue: error with remote domain #156

Closed KeyofBlueS closed 2 years ago

KeyofBlueS commented 2 years ago

Hi,

if I add a remote domain in a policy, i got an error, e.g.:

config policy
    option name 'example'
    option interface 'example_iface'
    option dest_addr 'google.com

Service Errors: iptables -t mangle -A PBR_PREROUTING -g PBR_MARK0x010000 -d 142.250.184.78,2a00:1450:4002:405::200e -m comment --comment example

if i try to run that command i got:

# iptables -t mangle -A PBR_PREROUTING -g PBR_MARK0x010000 -d 142.250.184.78,2a00:1450:4002:405::200e -m comment --comment example
iptables v1.8.7 (legacy): host/network `2a00:1450:4002:405::200e' not found
Try `iptables -h' or 'iptables --help' for more information.

I guess pbr is trying to apply ipv4 and ipv6 addresses with iptables command, while, i think, it should use iptables for ipv4 addresses and ip6tables for ipv6 addresses e.g.:

# iptables -t mangle -A PBR_PREROUTING -g PBR_MARK0x010000 -d 142.250.184.78 -m comment --comment example
# ip6tables -t mangle -A PBR_PREROUTING -g PBR_MARK0x010000 -d 2a00:1450:4002:405::200e -m comment --comment example

Tried with IPv6 Support both enabled and disabled, nothing changes.

pbr version 0.9.6-3

Sorry if i've not provided any requested output, but i'm not comfortable with that. If you really need, please ask me.

Thanks and best regards.

stangri commented 2 years ago

I'd highly recommend you switch to using resolver sets support for the policies with domains, but this particular issue has been fixed in pbr-iptables version 0.9.6-5. Please reopen if it's not fixed after updating the package.

KeyofBlueS commented 2 years ago

With version 0.9.6-5 that error is gone. The fact is, i was using dnsmasq ip set for domains yet. In any case, the domain added in the policy are not routed via the selected interface, i've flushed dns, rebooted the router and the local devices.

At this point i don't know if this issue should be reopened. Here there are my configurations, i've omitted some values. I hope it can be usefull for you.

Thanks.

  1. /etc/config/dhcp
config dnsmasq
    option domainneeded '1'
    option localise_queries '1'
    option rebind_protection '1'
    option rebind_localhost '1'
    option local '/lan/'
    option domain 'lan'
    option authoritative '1'
    option readethers '1'
    option leasefile '/tmp/dhcp.leases'
    option resolvfile '/tmp/resolv.conf.d/resolv.conf.auto'
    option localservice '1'
    option ednspacket_max '1232'
    option expandhosts '1'

config dhcp 'lan'
    option interface 'lan'
    option leasetime '12h'
    option dhcpv4 'server'
    list dhcp_option 'omissis'
    option force '1'
    option start '4'
    option limit '200'
    list ra_flags 'none'

config dhcp 'wan'
    option interface 'wan'
    option ignore '1'
    option start '100'
    option limit '150'
    option leasetime '12h'
    list dhcp_option 'omissis'
    list ra_flags 'none'

config odhcpd 'odhcpd'
    option maindhcp '0'
    option leasefile '/tmp/hosts/odhcpd'
    option leasetrigger '/usr/sbin/odhcpd-update'
    option loglevel '4'

  1. /etc/config/network

    
config interface 'loopback'
option device 'lo'
option proto 'static'
option ipaddr '127.0.0.1'
option netmask '255.0.0.0'

config globals 'globals'

config device option name 'br-lan' option type 'bridge' list ports 'eth0' option ipv6 '0'

config interface 'lan' option device 'br-lan' option proto 'static' option netmask '255.255.255.0' option ip6assign '60' option ipaddr '192.168.1.1' option ipv6 '0' option delegate '0' option dns 'omissis'

config interface 'wan' option device 'eth1' option ipv6 '0' option proto 'static' option ipaddr 'omissis' option netmask '255.255.255.0' option gateway 'omissis' option dns 'omissis'

config interface 'wan6' option device 'eth1' option proto 'dhcpv6'

config switch option name 'switch0' option reset '1' option enable_vlan '1'

config switch_vlan option device 'switch0' option vlan '1' option vid '1' option ports '0 1 2 3 4'

config device option name 'wlan2' option ipv6 '0'

config device option name 'wlan0' option ipv6 '0'

config device option name 'eth1' option ipv6 '0'

config device option name 'eth0' option ipv6 '0'

config interface 'vpn' option proto 'wireguard' option private_key 'omissis' list addresses '10.2.0.2/32' option defaultroute '0' option peerdns '0' list dns 'omissis'

config wireguard_vpn option public_key 'omissis' option endpoint_host 'omissis' option endpoint_port 'omissis' option persistent_keepalive '25' list allowed_ips '0.0.0.0/0' list allowed_ips '::0'


3. ```/etc/config/firewall```

config defaults option output 'ACCEPT' option forward 'REJECT' option synflood_protect '1' option input 'REJECT'

config zone 'lan' option name 'lan' option input 'ACCEPT' option output 'ACCEPT' option forward 'ACCEPT' list network 'lan'

config zone 'wan' option name 'wan' option input 'REJECT' option output 'ACCEPT' option forward 'REJECT' option masq '1' option mtu_fix '1' list network 'wan' list network 'wan6' list network 'vpn'

config forwarding option src 'lan' option dest 'wan'

config rule option name 'Allow-DHCP-Renew' option src 'wan' option proto 'udp' option dest_port '68' option target 'ACCEPT' option family 'ipv4'

config rule option name 'Allow-Ping' option src 'wan' option proto 'icmp' option icmp_type 'echo-request' option family 'ipv4' option target 'ACCEPT'

config rule option name 'Allow-IGMP' option src 'wan' option proto 'igmp' option family 'ipv4' option target 'ACCEPT'

config rule option name 'Allow-DHCPv6' option src 'wan' option proto 'udp' option src_ip 'fc00::/6' option dest_ip 'fc00::/6' option dest_port '546' option family 'ipv6' option target 'ACCEPT'

config rule option name 'Allow-MLD' option src 'wan' option proto 'icmp' option src_ip 'fe80::/10' list icmp_type '130/0' list icmp_type '131/0' list icmp_type '132/0' list icmp_type '143/0' option family 'ipv6' option target 'ACCEPT'

config rule option name 'Allow-ICMPv6-Input' option src 'wan' option proto 'icmp' list icmp_type 'echo-request' list icmp_type 'echo-reply' list icmp_type 'destination-unreachable' list icmp_type 'packet-too-big' list icmp_type 'time-exceeded' list icmp_type 'bad-header' list icmp_type 'unknown-header-type' list icmp_type 'router-solicitation' list icmp_type 'neighbour-solicitation' list icmp_type 'router-advertisement' list icmp_type 'neighbour-advertisement' option limit '1000/sec' option family 'ipv6' option target 'ACCEPT'

config rule option name 'Allow-ICMPv6-Forward' option src 'wan' option dest '*' option proto 'icmp' list icmp_type 'echo-request' list icmp_type 'echo-reply' list icmp_type 'destination-unreachable' list icmp_type 'packet-too-big' list icmp_type 'time-exceeded' list icmp_type 'bad-header' list icmp_type 'unknown-header-type' option limit '1000/sec' option family 'ipv6' option target 'ACCEPT'

config rule option name 'Allow-IPSec-ESP' option src 'wan' option dest 'lan' option proto 'esp' option target 'ACCEPT'

config rule option name 'Allow-ISAKMP' option src 'wan' option dest 'lan' option dest_port '500' option proto 'udp' option target 'ACCEPT'

config rule option name 'Support-UDP-Traceroute' option src 'wan' option dest_port '33434:33689' option proto 'udp' option family 'ipv4' option target 'REJECT' option enabled 'false'

config include option path '/etc/firewall.user'

config include 'pbr' option type 'script' option path '/usr/share/pbr/pbr.firewall.include'


4. ```/etc/config/pbr```

config policy option name 'misc' option interface 'vpn' option dest_addr 'ipleak.net'

config pbr 'config' option verbosity '2' option strict_enforcement '1' option src_ipset '0' option resolver_ipset 'dnsmasq.ipset' option iptables_rule_option 'append' option procd_reload_delay '1' option webui_show_ignore_target '0' option webui_sorting '1' list webui_supported_protocol 'tcp' list webui_supported_protocol 'udp' list webui_supported_protocol 'tcp udp' list webui_supported_protocol 'icmp' list webui_supported_protocol 'all' option dest_ipset '1' option webui_enable_column '1' option webui_protocol_column '1' option boot_timeout '60' list ignored_interface 'vpnserver wgserver' option rule_create_option 'add' option webui_chain_column '1' option ipv6_enabled '0' option resolver_set 'dnsmasq.ipset' option enabled '1'

config include option path '/etc/pbr.netflix.user' option enabled '0'

config include option path '/etc/pbr.aws.user' option enabled '0'


**PBR run-time information**

1. Output of ```/etc/init.d/pbr reload``` with ```verbosity``` set to 2:

[✓] Routing 'misc' via vpn [✓] pbr 0.9.6-5 started with gateways: wan/eth1/10.0.0.1 [✓] vpn/10.2.0.2


2. Output of ```/etc/init.d/pbr status -d```:

pbr 0.9.6-5 running on OpenWrt 21.02.3.

Dnsmasq version 2.85 Copyright (c) 2000-2021 Simon Kelley Compile time options: IPv6 GNU-getopt no-DBus UBus no-i18n no-IDN DHCP DHCPv6 no-Lua TFTP conntrack ipset auth cryptohash DNSSEC no-ID loop-detect inotify dumpfile

Routes/IP Rules Kernel IP routing table Destination Gateway Genmask Flags Metric Ref Use Iface default 10.0.0.1 0.0.0.0 UG 0 0 0 eth1 10.0.0.0 255.255.255.0 U 0 0 0 eth1 omissis 10.0.0.1 255.255.255.255 UGH 0 0 0 eth1 192.168.1.0 255.255.255.0 U 0 0 0 br-lan 0: from all lookup local

29999: from all fwmark 0x20000/0xff0000 lookup vpn 30000: from all fwmark 0x10000/0xff0000 lookup wan 32766: from all lookup main 32767: from all lookup default

IPv4 Table 201: default via 10.0.0.1 dev eth1 192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 IPv4 Table 201 Rules: 30000: from all fwmark 0x10000/0xff0000 lookup wan

IPv4 Table 202: default via 10.2.0.2 dev vpn 192.168.1.0/24 dev br-lan proto kernel scope link src 192.168.1.1 IPv4 Table 202 Rules: 29999: from all fwmark 0x20000/0xff0000 lookup vpn

Mangle IP Table Chain PREROUTING (policy ACCEPT) target prot opt source destination
PBR_PREROUTING all -- anywhere anywhere mark match 0x0/0xff0000

Chain INPUT (policy ACCEPT) target prot opt source destination

Chain FORWARD (policy ACCEPT) target prot opt source destination
TCPMSS tcp -- anywhere anywhere tcp flags:SYN,RST/SYN / !fw3: Zone wan MTU fixing / TCPMSS clamp to PMTU TCPMSS tcp -- anywhere anywhere tcp flags:SYN,RST/SYN / !fw3: Zone wan MTU fixing / TCPMSS clamp to PMTU TCPMSS tcp -- anywhere anywhere tcp flags:SYN,RST/SYN / !fw3: Zone wan MTU fixing / TCPMSS clamp to PMTU TCPMSS tcp -- anywhere anywhere tcp flags:SYN,RST/SYN / !fw3: Zone wan MTU fixing / TCPMSS clamp to PMTU TCPMSS tcp -- anywhere anywhere tcp flags:SYN,RST/SYN / !fw3: Zone wan MTU fixing / TCPMSS clamp to PMTU TCPMSS tcp -- anywhere anywhere tcp flags:SYN,RST/SYN / !fw3: Zone wan MTU fixing / TCPMSS clamp to PMTU TCPMSS tcp -- anywhere anywhere tcp flags:SYN,RST/SYN / !fw3: Zone wan MTU fixing / TCPMSS clamp to PMTU TCPMSS tcp -- anywhere anywhere tcp flags:SYN,RST/SYN / !fw3: Zone wan MTU fixing / TCPMSS clamp to PMTU

Chain OUTPUT (policy ACCEPT) target prot opt source destination

Chain POSTROUTING (policy ACCEPT) target prot opt source destination

Chain PBR_MARK0x010000 (0 references) target prot opt source destination
MARK all -- anywhere anywhere MARK xset 0x10000/0xff0000 RETURN all -- anywhere anywhere

Chain PBR_MARK0x020000 (0 references) target prot opt source destination
MARK all -- anywhere anywhere MARK xset 0x20000/0xff0000 RETURN all -- anywhere anywhere

Chain PBR_PREROUTING (1 references) target prot opt source destination
PBR_MARK0x020000 all -- anywhere 95.85.16.212 [goto] / misc /

Mangle IP Table MARK Chain: PBR_MARK0x010000 -N PBR_MARK0x010000 -A PBR_MARK0x010000 -c 0 0 -j MARK --set-xmark 0x10000/0xff0000 -A PBR_MARK0x010000 -c 0 0 -j RETURN

============================================================ Mangle IP Table MARK Chain: PBR_MARK0x020000 -N PBR_MARK0x020000 -A PBR_MARK0x020000 -c 9 468 -j MARK --set-xmark 0x20000/0xff0000 -A PBR_MARK0x020000 -c 9 468 -j RETURN

============================================================ NAT IP Table Chain PREROUTING (policy ACCEPT) target prot opt source destination
prerouting_rule all -- anywhere anywhere / !fw3: Custom prerouting rule chain / zone_lan_prerouting all -- anywhere anywhere / !fw3 / zone_wan_prerouting all -- anywhere anywhere / !fw3 / zone_wan_prerouting all -- anywhere anywhere / !fw3 / zone_wan_prerouting all -- anywhere anywhere / !fw3 / zone_wan_prerouting all -- anywhere anywhere / !fw3 /

Chain INPUT (policy ACCEPT) target prot opt source destination

Chain OUTPUT (policy ACCEPT) target prot opt source destination

Chain POSTROUTING (policy ACCEPT) target prot opt source destination
postrouting_rule all -- anywhere anywhere / !fw3: Custom postrouting rule chain / zone_lan_postrouting all -- anywhere anywhere / !fw3 / zone_wan_postrouting all -- anywhere anywhere / !fw3 / zone_wan_postrouting all -- anywhere anywhere / !fw3 / zone_wan_postrouting all -- anywhere anywhere / !fw3 / zone_wan_postrouting all -- anywhere anywhere / !fw3 /

Chain postrouting_lan_rule (1 references) target prot opt source destination

Chain postrouting_rule (1 references) target prot opt source destination

Chain postrouting_wan_rule (1 references) target prot opt source destination

Chain prerouting_lan_rule (1 references) target prot opt source destination

Chain prerouting_rule (1 references) target prot opt source destination

Chain prerouting_wan_rule (1 references) target prot opt source destination

Chain zone_lan_postrouting (1 references) target prot opt source destination
postrouting_lan_rule all -- anywhere anywhere / !fw3: Custom lan postrouting rule chain /

Chain zone_lan_prerouting (1 references) target prot opt source destination
prerouting_lan_rule all -- anywhere anywhere / !fw3: Custom lan prerouting rule chain /

Chain zone_wan_postrouting (4 references) target prot opt source destination
postrouting_wan_rule all -- anywhere anywhere / !fw3: Custom wan postrouting rule chain / MASQUERADE all -- anywhere anywhere / !fw3 /

Chain zone_wan_prerouting (4 references) target prot opt source destination
prerouting_wan_rule all -- anywhere anywhere / !fw3: Custom wan prerouting rule chain /

Current ipsets

Your support details have been logged to '/var/pbr-support'. [✓]

stangri commented 2 years ago

The pbr reload output seems to be truncated. Can you re-run that command and post the complete output please?

KeyofBlueS commented 2 years ago

It is the complete output, i've tried again right now, but it wasn't like this in version 0.9.6-3, i remember to see something like "creating table..."

KeyofBlueS commented 2 years ago

In order to trying to diagnose this issue, i've compared my configs with others in the issues section, and i've noticed that my pbr config had these options:

    option resolver_ipset 'dnsmasq.ipset'
    option iptables_rule_option 'append'
    option rule_create_option 'add'
    option resolver_set 'dnsmasq.ipset'

it looked weird to me and guessing there may be some misconfiguration, i've resetted and reconfigured pbr from scratch, and now is working as expected. From where resolver_ipset and iptables_rule_option came from? migrating from vpn-policy-routing maybe?

After that i've tried /etc/init.d/pbr reload and the output was complete, however subsequent attempts showed the truncated output.

Nevertheless the routing seems to work just fine, and the domain added in the policy are now routed via the selected interface.

You advised me to use resolver sets support for the policies with domains; how can i check if, even if correctly configured in pbr, i'm really using it?

Thanks!

stangri commented 2 years ago

it looked weird to me and guessing there may be some misconfiguration

The first two are legacy settings, in preparation to nft-compatible version I've had to remove any ipset/iptables from the option names.

You advised me to use resolver sets support for the policies with domains; how can i check if, even if correctly configured in pbr, i'm really using it?

When you run service pbr status you should see the section for DNSMASQ ipsets, it shouldn't be empty.

KeyofBlueS commented 2 years ago

Ok so in the pbr status log there is no DNSMASQ ipsets, what should i do to trackdown this issue? Do you need any log?

Thanks.

stangri commented 2 years ago

Ok so in the pbr status log there is no DNSMASQ ipsets, what should i do to trackdown this issue? Do you need any log?

Thanks.

Yes, all the files requested in README again would help.

KeyofBlueS commented 2 years ago

I think i've found what the problem is. After some experiments, i've found that the DNSMASQ ipsets were correctly created if, for a domain, i set the interface to wan, but not when set to vpn. The pbr script will fail in the ips funcion, command create_resolver, precisely at ipset -q -! create "$ipset" $param 2>/dev/null || failFlag=1, the exit code is 1. My vpn interface name is not "vpn", but a string of 14 characters (the maximum allowed through luci when creating a new interface). So when i tried the command below, omitting -q option: # ipset -! create "pbr_vpn_loong_name_dst_ip_cfg016ff5" hash:ip comment i get: ipset v7.6: Syntax error: setname 'pbr_vpn_loong_name_dst_ip_cfg016ff5' is longer than 31 characters

It's very late now, tomorrow i'll try to recreate the vpn interface giving it a shorter name to confirm, but i'm confident that the long name is the problem. If a way to manage this problem internally isn't possible, i advice to warn users to give an interface a name not longer than 10 characters, in the readme and in the error log if possible.

Thank you for your work and best regards :+1:

KeyofBlueS commented 2 years ago

Confirmed the issue was the name length. Taking into account that version 0.9.7-2 has the check for ipset/nft set name length, we can now close this.

Thanks again.