search

openwrt / packages

Community maintained packages for OpenWrt. Documentation for submitting pull requests is in CONTRIBUTING.md
GNU General Public License v2.0
3.94k stars 3.46k forks source link

mwan3: Multiwan switches to only 1 WAN only #23527

Open sophipl opened 7 months ago

sophipl commented 7 months ago

Environment: aarch64, AX9000, OpenWrt 23.05.2 r23630-842932a63d

Description: MWAN3 does not keep enabled more than one WAN. At bootup both 2 WANs are Online, I can send pings from them ping 8.8.8.8 -I vlan-plus and ping 8.8.8.8 -I vlan-tplus But shortly after one of the WAN is chosen, and another is disabled. There is no rule which one, it's random. In config file, there is another WAN, Orange, which is not working yet USB modem failsafe. Both interfaces keep their IPs

I tried many configurations, none works, replaced legacy xtables and tables to nft versions, i have no legacy packages installed on my system

Interface status:
 interface TPLUS is offline 00h:00m:00s, uptime 00h:26m:37s and tracking is active
 interface ORANGE is error (16) and tracking is paused
 interface PLUS is online 00h:08m:40s, uptime 00h:08m:41s and tracking is active

Sometimes the logs claims error 16 on one of TPLUS or PLUS too

cat /etc/config/mwan3

config globals 'globals'
        option mmx_mask '0x3F00'
        list rt_table_lookup '220'
        option logging '1'
        option loglevel 'notice'

config policy 'balanced'
        option last_resort 'unreachable'
        list use_member 'tplus_member'
        list use_member 'plus_member'
        list use_member 'orange_member'

config rule 'https'
        option sticky '1'
        option dest_port '443'
        option proto 'tcp'
        option use_policy 'balanced'

config rule 'default_rule_v4'
        option dest_ip '0.0.0.0/0'
        option use_policy 'balanced'
        option family 'ipv4'
        option proto 'all'
        option sticky '0'

config rule 'default_rule_v6'
        option dest_ip '::/0'
        option use_policy 'balanced'
        option family 'ipv6'

config interface 'TPLUS'
        option initial_state 'online'
        option family 'ipv4'
        option track_method 'ping'
        option reliability '1'
        option count '1'
        option timeout '4'
        option interval '10'
        option failure_interval '5'
        option recovery_interval '5'
        option down '5'
        option up '5'
        option enabled '1'
        list flush_conntrack 'connected'
        list track_ip '8.8.8.8'
        list track_ip '208.67.222.222'
        option size '56'
        option max_ttl '60'

config interface 'ORANGE'
        option enabled '1'
        option initial_state 'offline'
        option family 'ipv4'
        option track_method 'ping'
        option reliability '1'
        option count '1'
        option timeout '4'
        option interval '10'
        option failure_interval '5'
        option recovery_interval '5'
        option down '5'
        option up '5'
        list track_ip '8.8.4.4'
        list track_ip '208.67.220.220'
        option size '56'
        option max_ttl '60'

config interface 'PLUS'
        option enabled '1'
        option initial_state 'online'
        option family 'ipv4'
        option track_method 'ping'
        option reliability '1'
        option count '1'
        option timeout '4'
        option interval '10'
        option failure_interval '5'
        option recovery_interval '5'
        option down '5'
        option up '5'
        list flush_conntrack 'connected'
        list track_ip '8.8.8.8'
        list track_ip '208.67.220.220'
        option size '56'
        option max_ttl '60'

config policy 'orange_only'
        option last_resort 'unreachable'
        list use_member 'orange_member'

config rule 'inverter'
        option family 'ipv4'
        option proto 'all'
        option src_ip '10.234.12.100'
        option sticky '0'
        option use_policy 'orange_only'

config member 'plus_member'
        option interface 'PLUS'
        option metric '1'
        option weight '10'

config member 'tplus_member'
        option interface 'TPLUS'
        option metric '1'
        option weight '1000'

config member 'orange_member'
        option interface 'ORANGE'
        option metric '256'
        option weight '1'

part of cat /etc/config/network

config interface 'TPLUS'
        option proto 'dhcp'
        option device 'vlan-tplus'

config interface 'PLUS'
        option proto 'dhcp'
        option device 'vlan-plus'

config device
        option type '8021q'
        option ifname 'wan'
        option vid '23'
        option name 'vlan-tplus'

config device
        option type '8021q'
        option ifname 'wan'
        option vid '20'
        option name 'vlan-plus'

part of log

Sun Feb 25 00:35:17 2024 user.info mwan3track[2857]: Check (ping) failed for target "8.8.8.8" on interface TPLUS (vlan-tplus). Current score: 10
Sun Feb 25 00:35:21 2024 user.info mwan3track[2857]: Check (ping) failed for target "208.67.222.222" on interface TPLUS (vlan-tplus). Current score: 10
Sun Feb 25 00:35:21 2024 user.notice mwan3track[2857]: Interface TPLUS (vlan-tplus) is disconnecting
Sun Feb 25 00:35:31 2024 user.info mwan3track[2857]: Check (ping) failed for target "8.8.8.8" on interface TPLUS (vlan-tplus). Current score: 9
Sun Feb 25 00:35:35 2024 user.info mwan3track[2857]: Check (ping) failed for target "208.67.222.222" on interface TPLUS (vlan-tplus). Current score: 9
Sun Feb 25 00:35:44 2024 user.info mwan3track[2857]: Check (ping) failed for target "8.8.8.8" on interface TPLUS (vlan-tplus). Current score: 8
Sun Feb 25 00:35:48 2024 user.info mwan3track[2857]: Check (ping) failed for target "208.67.222.222" on interface TPLUS (vlan-tplus). Current score: 8
Sun Feb 25 00:35:57 2024 user.info mwan3track[2857]: Check (ping) failed for target "8.8.8.8" on interface TPLUS (vlan-tplus). Current score: 7
Sun Feb 25 00:36:01 2024 user.info mwan3track[2857]: Check (ping) failed for target "208.67.222.222" on interface TPLUS (vlan-tplus). Current score: 7
Sun Feb 25 00:36:10 2024 user.info mwan3track[2857]: Check (ping) failed for target "8.8.8.8" on interface TPLUS (vlan-tplus). Current score: 6
Sun Feb 25 00:36:14 2024 user.info mwan3track[2857]: Check (ping) failed for target "208.67.222.222" on interface TPLUS (vlan-tplus). Current score: 6
Sun Feb 25 00:36:14 2024 user.notice mwan3track[2857]: Interface TPLUS (vlan-tplus) is offline
brada4 commented 6 months ago

Can you attach nft list ruleset obfuscating ips to check if route marks are applied correct?

sophipl commented 6 months ago

Sure, keep in mind, I managed to boot Orange USB 3G WAN, it stays up, but it doesn't work, in a sence of wan rule for one device to use it. I needed to add static route to make it work

table inet fw4 {
        chain input {
                type filter hook input priority filter; policy drop;
                iifname "lo" accept comment "!fw4: Accept traffic from loopback"
                ct state established,related accept comment "!fw4: Allow inbound established and related flows"
                tcp flags syn / fin,syn,rst,ack jump syn_flood comment "!fw4: Rate limit TCP syn packets"
                iifname "br-lan" jump input_lan comment "!fw4: Handle lan IPv4/IPv6 input traffic"
                iifname "br-triv" jump input_TrivZone comment "!fw4: Handle TrivZone IPv4/IPv6 input traffic"
                iifname "br-work" jump input_WorkZone comment "!fw4: Handle WorkZone IPv4/IPv6 input traffic"
                iifname "vlan-tplus" jump input_TplusZone comment "!fw4: Handle TplusZone IPv4/IPv6 input traffic"
                iifname "vlan-plus" jump input_PlusZone comment "!fw4: Handle PlusZone IPv4/IPv6 input traffic"
                iifname "3g-ORANGE" jump input_OrangeZone comment "!fw4: Handle OrangeZone IPv4/IPv6 input traffic"
                iifname "br-nowan" jump input_NoWanZone comment "!fw4: Handle NoWanZone IPv4/IPv6 input traffic"
                jump handle_reject
        }

        chain forward {
                type filter hook forward priority filter; policy drop;
                ct state established,related accept comment "!fw4: Allow forwarded established and related flows"
                iifname "br-lan" jump forward_lan comment "!fw4: Handle lan IPv4/IPv6 forward traffic"
                iifname "br-triv" jump forward_TrivZone comment "!fw4: Handle TrivZone IPv4/IPv6 forward traffic"
                iifname "br-work" jump forward_WorkZone comment "!fw4: Handle WorkZone IPv4/IPv6 forward traffic"
                iifname "vlan-tplus" jump forward_TplusZone comment "!fw4: Handle TplusZone IPv4/IPv6 forward traffic"
                iifname "vlan-plus" jump forward_PlusZone comment "!fw4: Handle PlusZone IPv4/IPv6 forward traffic"
                iifname "3g-ORANGE" jump forward_OrangeZone comment "!fw4: Handle OrangeZone IPv4/IPv6 forward traffic"
                iifname "br-nowan" jump forward_NoWanZone comment "!fw4: Handle NoWanZone IPv4/IPv6 forward traffic"
                jump handle_reject
        }

        chain output {
                type filter hook output priority filter; policy accept;
                oifname "lo" accept comment "!fw4: Accept traffic towards loopback"
                ct state established,related accept comment "!fw4: Allow outbound established and related flows"
                oifname "br-lan" jump output_lan comment "!fw4: Handle lan IPv4/IPv6 output traffic"
                oifname "br-triv" jump output_TrivZone comment "!fw4: Handle TrivZone IPv4/IPv6 output traffic"
                oifname "br-work" jump output_WorkZone comment "!fw4: Handle WorkZone IPv4/IPv6 output traffic"
                oifname "vlan-tplus" jump output_TplusZone comment "!fw4: Handle TplusZone IPv4/IPv6 output traffic"
                oifname "vlan-plus" jump output_PlusZone comment "!fw4: Handle PlusZone IPv4/IPv6 output traffic"
                oifname "3g-ORANGE" jump output_OrangeZone comment "!fw4: Handle OrangeZone IPv4/IPv6 output traffic"
                oifname "br-nowan" jump output_NoWanZone comment "!fw4: Handle NoWanZone IPv4/IPv6 output traffic"
        }

        chain prerouting {
                type filter hook prerouting priority filter; policy accept;
                iifname "br-lan" jump helper_lan comment "!fw4: Handle lan IPv4/IPv6 helper assignment"
                iifname "br-triv" jump helper_TrivZone comment "!fw4: Handle TrivZone IPv4/IPv6 helper assignment"
                iifname "br-work" jump helper_WorkZone comment "!fw4: Handle WorkZone IPv4/IPv6 helper assignment"
                iifname "br-nowan" jump helper_NoWanZone comment "!fw4: Handle NoWanZone IPv4/IPv6 helper assignment"
        }

        chain handle_reject {
                meta l4proto tcp reject with tcp reset comment "!fw4: Reject TCP traffic"
                reject comment "!fw4: Reject any other traffic"
        }

        chain syn_flood {
                limit rate 25/second burst 50 packets return comment "!fw4: Accept SYN packets below rate-limit"
                drop comment "!fw4: Drop excess packets"
        }

        chain input_lan {
                jump accept_from_lan
        }

        chain output_lan {
                jump accept_to_lan
        }

        chain forward_lan {
                ip daddr IP1 counter packets 3 bytes 180 jump accept_to_TrivZone comment "!fw4: HomeAssistant"
                ip saddr IP2 counter packets 548 bytes 55979 jump accept_to_TrivZone comment "!fw4: @rule[12]"
                jump accept_to_TplusZone comment "!fw4: Accept lan to TplusZone forwarding"
                jump accept_to_PlusZone comment "!fw4: Accept lan to PlusZone forwarding"
                jump accept_to_OrangeZone comment "!fw4: Accept lan to OrangeZone forwarding"
                jump accept_to_TrivZone comment "!fw4: Accept lan to TrivZone forwarding"
                jump accept_to_lan
        }

        chain helper_lan {
        }

        chain accept_from_lan {
                iifname "br-lan" counter packets 648 bytes 46633 accept comment "!fw4: accept lan IPv4/IPv6 traffic"
        }

        chain accept_to_lan {
                oifname "br-lan" counter packets 835 bytes 50132 accept comment "!fw4: accept lan IPv4/IPv6 traffic"
        }

        chain input_TrivZone {
                jump accept_from_TrivZone
        }

        chain output_TrivZone {
                jump accept_to_TrivZone
        }

        chain forward_TrivZone {
                ip saddr IP1 counter packets 841 bytes 50460 jump accept_to_lan comment "!fw4: HomePiAccess"
                ip daddr IP3 udp dport 123 counter packets 0 bytes 0 jump accept_to_lan comment "!fw4: RecorderNTPAccess"
                ip daddr IP3 tcp dport 3142 counter packets 0 bytes 0 jump accept_to_lan comment "!fw4: RecorderCacherAccess"
                jump accept_to_TplusZone comment "!fw4: Accept TrivZone to TplusZone forwarding"
                jump accept_to_PlusZone comment "!fw4: Accept TrivZone to PlusZone forwarding"
                jump accept_to_OrangeZone comment "!fw4: Accept TrivZone to OrangeZone forwarding"
                jump accept_to_TrivZone
        }

        chain helper_TrivZone {
        }

        chain accept_from_TrivZone {
                iifname "br-triv" counter packets 224 bytes 24666 accept comment "!fw4: accept TrivZone IPv4/IPv6 traffic"
        }

        chain accept_to_TrivZone {
                oifname "br-triv" counter packets 550 bytes 54668 accept comment "!fw4: accept TrivZone IPv4/IPv6 traffic"
        }

        chain input_WorkZone {
                jump accept_from_WorkZone
        }

        chain output_WorkZone {
                jump accept_to_WorkZone
        }

        chain forward_WorkZone {
                jump accept_to_OrangeZone comment "!fw4: Accept WorkZone to OrangeZone forwarding"
                jump accept_to_PlusZone comment "!fw4: Accept WorkZone to PlusZone forwarding"
                jump accept_to_TplusZone comment "!fw4: Accept WorkZone to TplusZone forwarding"
                jump accept_to_WorkZone
        }

        chain helper_WorkZone {
        }

        chain accept_from_WorkZone {
                iifname "br-work" counter packets 0 bytes 0 accept comment "!fw4: accept WorkZone IPv4/IPv6 traffic"
        }

        chain accept_to_WorkZone {
                oifname "br-work" counter packets 0 bytes 0 accept comment "!fw4: accept WorkZone IPv4/IPv6 traffic"
        }

        chain input_TplusZone {
                meta nfproto ipv4 udp dport 68 counter packets 0 bytes 0 accept comment "!fw4: Allow-DHCP-Renew"
                jump reject_from_TplusZone
        }

        chain output_TplusZone {
                jump accept_to_TplusZone
        }

        chain forward_TplusZone {
                jump reject_to_TplusZone
        }

        chain accept_to_TplusZone {
                meta nfproto ipv4 oifname "vlan-tplus" ct state invalid counter packets 92 bytes 4680 drop comment "!fw4: Prevent NAT leakage"
                oifname "vlan-tplus" counter packets 526 bytes 46834 accept comment "!fw4: accept TplusZone IPv4/IPv6 traffic"
        }

        chain reject_from_TplusZone {
                iifname "vlan-tplus" counter packets 97 bytes 7679 jump handle_reject comment "!fw4: reject TplusZone IPv4/IPv6 traffic"
        }

        chain reject_to_TplusZone {
                oifname "vlan-tplus" counter packets 0 bytes 0 jump handle_reject comment "!fw4: reject TplusZone IPv4/IPv6 traffic"
        }

        chain input_PlusZone {
                meta nfproto ipv4 udp dport 68 counter packets 0 bytes 0 accept comment "!fw4: Allow-DHCP-Renew"
                jump reject_from_PlusZone
        }

        chain output_PlusZone {
                jump accept_to_PlusZone
        }

        chain forward_PlusZone {
                jump reject_to_PlusZone
        }

        chain accept_to_PlusZone {
                meta nfproto ipv4 oifname "vlan-plus" ct state invalid counter packets 0 bytes 0 drop comment "!fw4: Prevent NAT leakage"
                oifname "vlan-plus" counter packets 160 bytes 19420 accept comment "!fw4: accept PlusZone IPv4/IPv6 traffic"
        }

        chain reject_from_PlusZone {
                iifname "vlan-plus" counter packets 4 bytes 128 jump handle_reject comment "!fw4: reject PlusZone IPv4/IPv6 traffic"
        }

        chain reject_to_PlusZone {
                oifname "vlan-plus" counter packets 0 bytes 0 jump handle_reject comment "!fw4: reject PlusZone IPv4/IPv6 traffic"
        }

        chain input_OrangeZone {
                meta nfproto ipv4 udp dport 68 counter packets 0 bytes 0 accept comment "!fw4: Allow-DHCP-Renew"
                jump reject_from_OrangeZone
        }

        chain output_OrangeZone {
                jump accept_to_OrangeZone
        }

        chain forward_OrangeZone {
                jump reject_to_OrangeZone
        }

        chain accept_to_OrangeZone {
                meta nfproto ipv4 oifname "3g-ORANGE" ct state invalid counter packets 0 bytes 0 drop comment "!fw4: Prevent NAT leakage"
                oifname "3g-ORANGE" counter packets 72 bytes 5240 accept comment "!fw4: accept OrangeZone IPv4/IPv6 traffic"
        }

        chain reject_from_OrangeZone {
                iifname "3g-ORANGE" counter packets 0 bytes 0 jump handle_reject comment "!fw4: reject OrangeZone IPv4/IPv6 traffic"
        }

        chain reject_to_OrangeZone {
                oifname "3g-ORANGE" counter packets 0 bytes 0 jump handle_reject comment "!fw4: reject OrangeZone IPv4/IPv6 traffic"
        }

        chain input_NoWanZone {
                jump accept_from_NoWanZone
        }

        chain output_NoWanZone {
                jump accept_to_NoWanZone
        }

        chain forward_NoWanZone {
                jump reject_to_NoWanZone
        }

        chain helper_NoWanZone {
        }

        chain accept_from_NoWanZone {
                iifname "br-nowan" counter packets 10 bytes 757 accept comment "!fw4: accept NoWanZone IPv4/IPv6 traffic"
        }

        chain accept_to_NoWanZone {
                oifname "br-nowan" counter packets 0 bytes 0 accept comment "!fw4: accept NoWanZone IPv4/IPv6 traffic"
        }

        chain reject_to_NoWanZone {
                oifname "br-nowan" counter packets 0 bytes 0 jump handle_reject comment "!fw4: reject NoWanZone IPv4/IPv6 traffic"
        }

        chain dstnat {
                type nat hook prerouting priority dstnat; policy accept;
        }

        chain srcnat {
                type nat hook postrouting priority srcnat; policy accept;
                oifname "vlan-tplus" jump srcnat_TplusZone comment "!fw4: Handle TplusZone IPv4/IPv6 srcnat traffic"
                oifname "vlan-plus" jump srcnat_PlusZone comment "!fw4: Handle PlusZone IPv4/IPv6 srcnat traffic"
                oifname "3g-ORANGE" jump srcnat_OrangeZone comment "!fw4: Handle OrangeZone IPv4/IPv6 srcnat traffic"
        }

        chain srcnat_TplusZone {
                meta nfproto ipv4 masquerade comment "!fw4: Masquerade IPv4 TplusZone traffic"
        }

        chain srcnat_PlusZone {
                meta nfproto ipv4 masquerade comment "!fw4: Masquerade IPv4 PlusZone traffic"
        }

        chain srcnat_OrangeZone {
                meta nfproto ipv4 masquerade comment "!fw4: Masquerade IPv4 OrangeZone traffic"
        }

        chain raw_prerouting {
                type filter hook prerouting priority raw; policy accept;
        }

        chain raw_output {
                type filter hook output priority raw; policy accept;
        }

        chain mangle_prerouting {
                type filter hook prerouting priority mangle; policy accept;
        }

        chain mangle_postrouting {
                type filter hook postrouting priority mangle; policy accept;
        }

        chain mangle_input {
                type filter hook input priority mangle; policy accept;
        }

        chain mangle_output {
                type route hook output priority mangle; policy accept;
        }

        chain mangle_forward {
                type filter hook forward priority mangle; policy accept;
                iifname "vlan-tplus" tcp flags syn tcp option maxseg size set rt mtu comment "!fw4: Zone TplusZone IPv4/IPv6 ingress MTU fixing"
                oifname "vlan-tplus" tcp flags syn tcp option maxseg size set rt mtu comment "!fw4: Zone TplusZone IPv4/IPv6 egress MTU fixing"
        }
}
# Warning: table ip mangle is managed by iptables-nft, do not touch!
table ip mangle {
        chain mwan3_ifaces_in {
                meta mark & 0x00003f00 == 0x00000000 counter packets 13202 bytes 1439919 jump mwan3_iface_in_ORANGE
                meta mark & 0x00003f00 == 0x00000000 counter packets 13107 bytes 1405258 jump mwan3_iface_in_PLUS
                meta mark & 0x00003f00 == 0x00000000 counter packets 13056 bytes 1401321 jump mwan3_iface_in_TPLUS
        }

        chain mwan3_custom_ipv4 {
                xt match "set" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_connected_ipv4 {
                xt match "set" counter packets 1654132 bytes 2380078085 xt target "MARK"
        }

        chain mwan3_dynamic_ipv4 {
                xt match "set" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_rules {
                meta l4proto tcp xt match "multiport" meta mark & 0x00003f00 == 0x00000000 counter packets 655 bytes 60552 jump mwan3_rule_https
                meta mark & 0x00003f00 == 0x00000000 counter packets 2423 bytes 375800 jump mwan3_policy_balanced
                ip saddr IP10 meta mark & 0x00003f00 == 0x00000000 counter packets 0 bytes 0 jump mwan3_rule_inverter
                ip saddr IP2 meta mark & 0x00003f00 == 0x00000000 counter packets 0 bytes 0 jump mwan3_rule_debian
        }

        chain mwan3_hook {
                meta mark & 0x00003f00 == 0x00000000 counter packets 2615603 bytes 2509036497 xt target "CONNMARK"
                meta mark & 0x00003f00 == 0x00000000 counter packets 13252 bytes 1451612 jump mwan3_ifaces_in
                meta mark & 0x00003f00 == 0x00000000 counter packets 12444 bytes 1249518 jump mwan3_custom_ipv4
                meta mark & 0x00003f00 == 0x00000000 counter packets 12444 bytes 1249518 jump mwan3_connected_ipv4
                meta mark & 0x00003f00 == 0x00000000 counter packets 3078 bytes 436352 jump mwan3_dynamic_ipv4
                meta mark & 0x00003f00 == 0x00000000 counter packets 3078 bytes 436352 jump mwan3_rules
                counter packets 2655224 bytes 2512245563 xt target "CONNMARK"
                meta mark & 0x00003f00 != 0x00003f00 counter packets 2454785 bytes 2436741475 jump mwan3_custom_ipv4
                meta mark & 0x00003f00 != 0x00003f00 counter packets 2454785 bytes 2436741475 jump mwan3_connected_ipv4
                meta mark & 0x00003f00 != 0x00003f00 counter packets 810019 bytes 57476556 jump mwan3_dynamic_ipv4
        }

        chain PREROUTING {
                type filter hook prerouting priority mangle; policy accept;
                counter packets 2576212 bytes 2478371641 jump mwan3_hook
        }

        chain OUTPUT {
                type route hook output priority mangle; policy accept;
                counter packets 79012 bytes 33873922 jump mwan3_hook
        }

        chain mwan3_policy_balanced {
                meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 347 bytes 30640 xt target "MARK"
        }

        chain mwan3_policy_orange_only {
                meta mark & 0x00003f00 == 0x00000000 xt match "statistic" xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_policy_plus_only {
                oifname "vlan-plus" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_policy_tplus_only {
                meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_rule_https {
                meta mark & 0x00003f00 == 0x00000000 counter packets 655 bytes 60552 jump mwan3_policy_balanced
                meta mark & 0x0000fc00 != 0x0000fc00 counter packets 655 bytes 60552 xt target "SET"
                meta mark & 0x0000fc00 != 0x0000fc00 counter packets 655 bytes 60552 xt target "SET"
        }

        chain mwan3_rule_inverter {
                meta mark & 0x00003f00 == 0x00000000 counter packets 0 bytes 0 jump mwan3_policy_orange_only
                meta mark & 0x0000fc00 != 0x0000fc00 counter packets 0 bytes 0 xt target "SET"
                meta mark & 0x0000fc00 != 0x0000fc00 counter packets 0 bytes 0 xt target "SET"
        }

        chain mwan3_rule_debian {
                meta mark & 0x00003f00 == 0x00000000 counter packets 0 bytes 0 jump mwan3_policy_orange_only
                meta mark & 0x0000fc00 != 0x0000fc00 counter packets 0 bytes 0 xt target "SET"
                meta mark & 0x0000fc00 != 0x0000fc00 counter packets 0 bytes 0 xt target "SET"
        }

        chain mwan3_iface_in_ORANGE {
                iifname "3g-ORANGE" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                iifname "3g-ORANGE" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                iifname "3g-ORANGE" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                iifname "3g-ORANGE" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 2 bytes 80 xt target "MARK"
        }

        chain mwan3_iface_in_PLUS {
                iifname "vlan-plus" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                iifname "vlan-plus" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                iifname "vlan-plus" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                iifname "vlan-plus" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 4 bytes 128 xt target "MARK"
        }

        chain mwan3_iface_in_TPLUS {
                iifname "vlan-tplus" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                iifname "vlan-tplus" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 100 bytes 33244 xt target "MARK"
                iifname "vlan-tplus" xt match "set" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
                iifname "vlan-tplus" meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 83 bytes 6687 xt target "MARK"
        }
}
# Warning: table ip6 mangle is managed by iptables-nft, do not touch!
table ip6 mangle {
        chain mwan3_ifaces_in {
        }

        chain mwan3_custom_ipv6 {
                xt match "set" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_connected_ipv6 {
                xt match "set" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_dynamic_ipv6 {
                xt match "set" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_rules {
                meta l4proto tcp xt match "multiport" meta mark & 0x00003f00 == 0x00000000 counter packets 0 bytes 0 jump mwan3_rule_https
                meta mark & 0x00003f00 == 0x00000000 counter packets 746 bytes 138970 jump mwan3_policy_balanced
        }

        chain mwan3_hook {
                meta l4proto ipv6-icmp xt match "icmp6" counter packets 72 bytes 3992 return
                meta l4proto ipv6-icmp xt match "icmp6" counter packets 10 bytes 640 return
                meta l4proto ipv6-icmp xt match "icmp6" counter packets 39 bytes 2712 return
                meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 return
                meta l4proto ipv6-icmp xt match "icmp6" counter packets 0 bytes 0 return
                meta mark & 0x00003f00 == 0x00000000 counter packets 817 bytes 151506 xt target "CONNMARK"
                meta mark & 0x00003f00 == 0x00000000 counter packets 746 bytes 138970 jump mwan3_ifaces_in
                meta mark & 0x00003f00 == 0x00000000 counter packets 746 bytes 138970 jump mwan3_custom_ipv6
                meta mark & 0x00003f00 == 0x00000000 counter packets 746 bytes 138970 jump mwan3_connected_ipv6
                meta mark & 0x00003f00 == 0x00000000 counter packets 746 bytes 138970 jump mwan3_dynamic_ipv6
                meta mark & 0x00003f00 == 0x00000000 counter packets 746 bytes 138970 jump mwan3_rules
                counter packets 913 bytes 166093 xt target "CONNMARK"
                meta mark & 0x00003f00 != 0x00003f00 counter packets 913 bytes 166093 jump mwan3_custom_ipv6
                meta mark & 0x00003f00 != 0x00003f00 counter packets 913 bytes 166093 jump mwan3_connected_ipv6
                meta mark & 0x00003f00 != 0x00003f00 counter packets 913 bytes 166093 jump mwan3_dynamic_ipv6
        }

        chain PREROUTING {
                type filter hook prerouting priority mangle; policy accept;
                counter packets 802 bytes 146146 jump mwan3_hook
        }

        chain OUTPUT {
                type route hook output priority mangle; policy accept;
                counter packets 232 bytes 27291 jump mwan3_hook
        }

        chain mwan3_policy_balanced {
                meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 60 bytes 10495 xt target "MARK"
        }

        chain mwan3_policy_orange_only {
                meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_policy_plus_only {
                meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_policy_tplus_only {
                meta mark & 0x00003f00 == 0x00000000 xt match "comment" counter packets 0 bytes 0 xt target "MARK"
        }

        chain mwan3_rule_https {
                meta mark & 0x00003f00 == 0x00000000 counter packets 0 bytes 0 jump mwan3_policy_balanced
                meta mark & 0x0000fc00 != 0x0000fc00 counter packets 0 bytes 0 xt target "SET"
                meta mark & 0x0000fc00 != 0x0000fc00 counter packets 0 bytes 0 xt target "SET"
        }
}
brada4 commented 6 months ago

Normally iptables-nft does not offfload to xtable mods. you are missing kmod-nf-conntrack6, then reboot and check/post if any other xt matches remain. Most likely xt mark does damage.

sophipl commented 6 months ago

I'm afraid I already have this package That's my list-installed

airmon-ng - 1.7-1
ath10k-board-qca9887 - 20230804-1
ath10k-firmware-qca9887-ct - 2020-11-08-1
ath11k-firmware-ipq8074 - 2023-03-31-a039049a-1
ath11k-firmware-qcn9074 - 2023-03-31-a039049a-1
base-files - 1550-r23630-842932a63d
busybox - 1.36.1-1
ca-bundle - 20230311-1
cgi-io - 2022-08-10-901b0f04-21
chat - 2.4.9.git-2021-01-04-4
collectd - 5.12.0-49
collectd-mod-cpu - 5.12.0-49
collectd-mod-interface - 5.12.0-49
collectd-mod-iwinfo - 5.12.0-49
collectd-mod-load - 5.12.0-49
collectd-mod-memory - 5.12.0-49
collectd-mod-network - 5.12.0-49
collectd-mod-rrdtool - 5.12.0-49
comgt - 0.32-35
comgt-ncm - 0.32-35
curl - 8.6.0-1
ddns-scripts - 2.8.2-42
ddns-scripts-cloudflare - 2.8.2-42
ddns-scripts-services - 2.8.2-42
dnsmasq - 2.89-4
dropbear - 2022.82-5
e2fsprogs - 1.47.0-2
etherwake - 1.09-5
ethtool - 6.3-1
firewall4 - 2023-09-01-598d9fbb-1
fstools - 2023-02-28-bfe882d5-1
fwtool - 2019-11-12-8f7fe925-1
getrandom - 2022-08-13-4c7b720b-2
hostapd-common - 2023-09-08-e5ccbfc6-6
htop - 3.3.0-1
httping - 2.5-1
ip-bridge - 6.3.0-1
ip-tiny - 6.3.0-1
ip6tables-nft - 1.8.8-2
ipq-wifi-xiaomi_ax9000 - 2023-11-10-0c2e810e-1
ipset - 7.17-1
iptables-mod-conntrack-extra - 1.8.8-2
iptables-mod-ipopt - 1.8.8-2
iptables-nft - 1.8.8-2
iw - 5.19-1
iwinfo - 2023-07-01-ca79f641-1
jansson4 - 2.14-3
jshn - 2023-05-23-75a3b870-1
jsonfilter - 2018-02-04-c7e938d6-1
kernel - 5.15.137-1-c0be4d8060b09729c42faeda72adef10
kmod-ath - 5.15.137+6.1.24-3
kmod-ath10k-ct - 5.15.137+2022-05-13-f808496f-5
kmod-ath11k - 5.15.137+6.1.24-3
kmod-ath11k-ahb - 5.15.137+6.1.24-3
kmod-ath11k-pci - 5.15.137+6.1.24-3
kmod-cfg80211 - 5.15.137+6.1.24-3
kmod-crypto-acompress - 5.15.137-1
kmod-crypto-aead - 5.15.137-1
kmod-crypto-ccm - 5.15.137-1
kmod-crypto-cmac - 5.15.137-1
kmod-crypto-crc32c - 5.15.137-1
kmod-crypto-ctr - 5.15.137-1
kmod-crypto-gcm - 5.15.137-1
kmod-crypto-gf128 - 5.15.137-1
kmod-crypto-ghash - 5.15.137-1
kmod-crypto-hash - 5.15.137-1
kmod-crypto-hmac - 5.15.137-1
kmod-crypto-manager - 5.15.137-1
kmod-crypto-michael-mic - 5.15.137-1
kmod-crypto-null - 5.15.137-1
kmod-crypto-rng - 5.15.137-1
kmod-crypto-seqiv - 5.15.137-1
kmod-crypto-sha512 - 5.15.137-1
kmod-fs-ext4 - 5.15.137-1
kmod-gpio-button-hotplug - 5.15.137-3
kmod-hwmon-core - 5.15.137-1
kmod-ip6tables - 5.15.137-1
kmod-ipt-conntrack - 5.15.137-1
kmod-ipt-conntrack-extra - 5.15.137-1
kmod-ipt-core - 5.15.137-1
kmod-ipt-ipopt - 5.15.137-1
kmod-ipt-ipset - 5.15.137-1
kmod-leds-gpio - 5.15.137-1
kmod-lib-crc-ccitt - 5.15.137-1
kmod-lib-crc16 - 5.15.137-1
kmod-lib-crc32c - 5.15.137-1
kmod-lib-lzo - 5.15.137-1
kmod-libphy - 5.15.137-1
kmod-mac80211 - 5.15.137+6.1.24-3
kmod-mhi-bus - 5.15.137-1
kmod-mii - 5.15.137-1
kmod-nf-conncount - 5.15.137-1
kmod-nf-conntrack - 5.15.137-1
kmod-nf-conntrack6 - 5.15.137-1
kmod-nf-flow - 5.15.137-1
kmod-nf-ipt - 5.15.137-1
kmod-nf-ipt6 - 5.15.137-1
kmod-nf-log - 5.15.137-1
kmod-nf-log6 - 5.15.137-1
kmod-nf-nat - 5.15.137-1
kmod-nf-reject - 5.15.137-1
kmod-nf-reject6 - 5.15.137-1
kmod-nfnetlink - 5.15.137-1
kmod-nft-compat - 5.15.137-1
kmod-nft-core - 5.15.137-1
kmod-nft-fib - 5.15.137-1
kmod-nft-nat - 5.15.137-1
kmod-nft-offload - 5.15.137-1
kmod-nls-base - 5.15.137-1
kmod-phy-aquantia - 5.15.137-1
kmod-ppp - 5.15.137-1
kmod-pppoe - 5.15.137-1
kmod-pppox - 5.15.137-1
kmod-qca-nss-dp - 5.15.137+2022-04-30-72e9ec41-1
kmod-qca-ssdk - 5.15.137+2022-09-12-628b22bc-2
kmod-qrtr - 5.15.137-1
kmod-qrtr-mhi - 5.15.137-1
kmod-qrtr-smd - 5.15.137-1
kmod-slhc - 5.15.137-1
kmod-thermal - 5.15.137-1
kmod-usb-core - 5.15.137-1
kmod-usb-dwc3 - 5.15.137-1
kmod-usb-dwc3-qcom - 5.15.137-1
kmod-usb-ehci - 5.15.137-1
kmod-usb-net - 5.15.137-1
kmod-usb-net-cdc-ether - 5.15.137-1
kmod-usb-net-cdc-ncm - 5.15.137-1
kmod-usb-net-huawei-cdc-ncm - 5.15.137-1
kmod-usb-net-qmi-wwan - 5.15.137-1
kmod-usb-ohci - 5.15.137-1
kmod-usb-ohci-pci - 5.15.137-1
kmod-usb-serial - 5.15.137-1
kmod-usb-serial-option - 5.15.137-1
kmod-usb-serial-wwan - 5.15.137-1
kmod-usb-wdm - 5.15.137-1
kmod-usb-xhci-hcd - 5.15.137-1
kmod-usb2 - 5.15.137-1
kmod-usb2-pci - 5.15.137-1
kmod-usb3 - 5.15.137-1
kmod-wwan - 5.15.137-1
libatomic1 - 12.3.0-4
libblkid1 - 2.39-2
libblobmsg-json20230523 - 2023-05-23-75a3b870-1
libc - 1.2.4-4
libcomerr0 - 1.47.0-2
libcurl4 - 8.6.0-1
libevdev - 1.13.0-1
libext2fs2 - 1.47.0-2
libgcc1 - 12.3.0-4
libip4tc2 - 1.8.8-2
libip6tc2 - 1.8.8-2
libipset13 - 7.17-1
libiptext-nft0 - 1.8.8-2
libiptext0 - 1.8.8-2
libiptext6-0 - 1.8.8-2
libiwinfo-data - 2023-07-01-ca79f641-1
libiwinfo20230701 - 2023-07-01-ca79f641-1
libjson-c5 - 0.16-3
libjson-script20230523 - 2023-05-23-75a3b870-1
libltdl7 - 2.4.7-1
liblua5.1.5 - 5.1.5-10
liblucihttp-lua - 2023-03-15-9b5b683f-1
liblucihttp-ucode - 2023-03-15-9b5b683f-1
liblucihttp0 - 2023-03-15-9b5b683f-1
libmbedtls12 - 2.28.5-2
libmnl0 - 1.0.5-1
libncurses6 - 6.4-2
libnftnl11 - 1.2.6-1
libnghttp2-14 - 1.57.0-1
libnl-tiny1 - 2023-07-27-bc92a280-1
libopenssl3 - 3.0.13-1
libpcap1 - 1.10.4-1
libpopt0 - 1.19-1
libpthread - 1.2.4-4
librrd1 - 1.0.50-5
librt - 1.2.4-4
libsmartcols1 - 2.39-2
libss2 - 1.47.0-2
libubox20230523 - 2023-05-23-75a3b870-1
libubus-lua - 2023-06-05-f787c97b-1
libubus20230605 - 2023-06-05-f787c97b-1
libuci20130104 - 2023-08-10-5781664d-1
libuclient20201210 - 2023-04-13-007d9454-1
libucode20230711 - 2023-11-07-a6e75e02-1
libudev-zero - 1.0.1-1
libusb-1.0-0 - 1.0.26-3
libustream-mbedtls20201210 - 2023-02-25-498f6e26-1
libuuid1 - 2.39-2
libxtables12 - 1.8.8-2
logd - 2022-08-13-4c7b720b-2
losetup - 2.39-2
lua - 5.1.5-10
luci - git-23.051.66410-a505bb1
luci-app-commands - git-22.299.76611-9d8feac
luci-app-ddns - git-23.346.52990-28c4a65
luci-app-diag-core - git-20.036.57178-625abbf
luci-app-firewall - git-23.306.38853-a0466cd
luci-app-mwan3 - git-22.181.29827-675a0ea
luci-app-openvpn - git-24.039.49284-d5c562e
luci-app-opkg - git-23.311.75635-769b30c
luci-app-statistics - git-24.034.36441-feee897
luci-app-wol - git-21.154.28269-c7b7b42
luci-base - git-23.306.39416-c86c256
luci-compat - git-23.351.85440-817a5e6
luci-lib-base - git-22.308.54612-9118452
luci-lib-ip - git-23.311.79290-c2a887e
luci-lib-jsonc - git-23.298.74571-62eb535
luci-lib-nixio - git-24.034.54875-21210dc
luci-light - git-23.024.33244-34dee82
luci-lua-runtime - git-23.233.52805-dae2684
luci-mod-admin-full - git-19.253.48496-3f93650
luci-mod-dashboard - git-23.306.38853-672fd9d
luci-mod-network - git-23.313.56166-6da284d
luci-mod-status - git-23.306.52197-bdcd3e0
luci-mod-system - git-23.306.39416-7d3abf8
luci-proto-3g - git-21.231.25157-5ff3ef7
luci-proto-ipv6 - git-21.148.48881-79947af
luci-proto-ncm - git-22.209.60806-049d876
luci-proto-ppp - git-21.158.38888-88b9d84
luci-proto-qmi - git-21.231.25157-5ff3ef7
luci-ssl - git-23.035.26083-7550ad6
luci-theme-bootstrap - git-23.338.81541-94798fc
luci-theme-material - git-23.088.30860-f464199
luci-theme-openwrt - git-23.009.43662-21b4065
luci-theme-openwrt-2020 - git-24.011.08873-8f75be4
mtd - 26
mwan3 - 2.11.8-1
netifd - 2023-11-10-35facc83-1.1
nftables-json - 1.0.8-1
odhcp6c - 2023-05-12-bcd28363-20
odhcpd-ipv6only - 2023-10-24-d8118f6e-1
openwrt-keyring - 2022-03-25-62471e69-2
opkg - 2022-02-24-d038e5b6-2
picocom - 3.1-5
ppp - 2.4.9.git-2021-01-04-4
ppp-mod-pppoe - 2.4.9.git-2021-01-04-4
procd - 2023-06-25-2db83655-2
procd-seccomp - 2023-06-25-2db83655-2
procd-ujail - 2023-06-25-2db83655-2
procps-ng - 3.3.16-3
px5g-mbedtls - 10
rpcd - 2023-07-01-c07ab2f9-1
rpcd-mod-file - 2023-07-01-c07ab2f9-1
rpcd-mod-iwinfo - 2023-07-01-c07ab2f9-1
rpcd-mod-luci - 20230123-1
rpcd-mod-rrdns - 20170710
rpcd-mod-ucode - 2023-07-01-c07ab2f9-1
rrdtool1 - 1.0.50-5
rsync - 3.2.7-1
sms-tool - 2022-03-21-f07699ab-1
tcpdump - 4.99.4-1
terminfo - 6.4-2
ubi-utils - 2.1.5-1
uboot-envtools - 2023.04-1
ubox - 2022-08-13-4c7b720b-2
ubus - 2023-06-05-f787c97b-1
ubusd - 2023-06-05-f787c97b-1
uci - 2023-08-10-5781664d-1
uclient-fetch - 2023-04-13-007d9454-1
ucode - 2023-11-07-a6e75e02-1
ucode-mod-fs - 2023-11-07-a6e75e02-1
ucode-mod-html - 1
ucode-mod-lua - 1
ucode-mod-math - 2023-11-07-a6e75e02-1
ucode-mod-nl80211 - 2023-11-07-a6e75e02-1
ucode-mod-rtnl - 2023-11-07-a6e75e02-1
ucode-mod-ubus - 2023-11-07-a6e75e02-1
ucode-mod-uci - 2023-11-07-a6e75e02-1
ucode-mod-uloop - 2023-11-07-a6e75e02-1
uhttpd - 2023-06-25-34a8a74d-1
uhttpd-mod-ubus - 2023-06-25-34a8a74d-1
uqmi - 2022-10-20-c8c9f105-1
urandom-seed - 3
urngd - 2023-11-01-44365eb1-1
usb-modeswitch - 2022-02-24-3c8595a4-1
usbids - 0.359-1
usbutils - 014-1
usign - 2020-05-23-f1f65026-1
wireless-regdb - 2023.09.01-1
wireless-tools - 29-6
wpad-basic-mbedtls - 2023-09-08-e5ccbfc6-6
wwan - 2019-04-29-6
xtables-nft - 1.8.8-2
zlib - 1.2.13-1
brada4 commented 6 months ago

xt marks and nft marks dont see eachother. @jow- might be better versed which module got skipped.