johang / sd-card-images

Scripts to build bootable SD card images with Debian for various single-board computers
https://sd-card-images.johang.se
GNU General Public License v3.0
158 stars 35 forks source link

Increasingly slow ethernet after boot on Orange Pi R1 Plus LTS #101

Open DaveWK opened 10 months ago

DaveWK commented 10 months ago

Hi, I have been checking out the images for the Orange Pi R1 Plus LTS, and after it finishes booting, when I attempt to do anything that requires network connectivity such as running apt update, within a few seconds the connection gets increasingly slow to the point it no longer is functional. The YT8531 is detected by all of the images:

[   14.527865] usbcore: registered new interface driver cdc_ether
[   14.708545] rk_gmac-dwmac ff540000.ethernet: IRQ eth_wake_irq not found
[   14.709215] rk_gmac-dwmac ff540000.ethernet: IRQ eth_lpi not found
[   14.743766] rk_gmac-dwmac ff540000.ethernet: PTP uses main clock
[   14.744949] rk_gmac-dwmac ff540000.ethernet: clock input or output? (input).
[   14.745824] rk_gmac-dwmac ff540000.ethernet: TX delay(0x19).
[   14.746376] rk_gmac-dwmac ff540000.ethernet: RX delay(0x5).
[   14.746917] rk_gmac-dwmac ff540000.ethernet: integrated PHY? (no).
[   14.752152] rk_gmac-dwmac ff540000.ethernet: clock input from PHY
[   14.757829] rk_gmac-dwmac ff540000.ethernet: init for RGMII
[   14.768972] rk_gmac-dwmac ff540000.ethernet: User ID: 0x10, Synopsys ID: 0x35
[   14.769745] rk_gmac-dwmac ff540000.ethernet:     DWMAC1000
[   14.770250] rk_gmac-dwmac ff540000.ethernet: DMA HW capability register supported
[   14.770941] rk_gmac-dwmac ff540000.ethernet: RX Checksum Offload Engine supported
[   14.771703] rk_gmac-dwmac ff540000.ethernet: COE Type 2
[   14.772201] rk_gmac-dwmac ff540000.ethernet: TX Checksum insertion supported
[   14.773027] rk_gmac-dwmac ff540000.ethernet: Wake-Up On Lan supported
[   14.774117] rk_gmac-dwmac ff540000.ethernet: Normal descriptors
[   14.774714] rk_gmac-dwmac ff540000.ethernet: Ring mode enabled
[   14.775262] rk_gmac-dwmac ff540000.ethernet: Enable RX Mitigation via HW Watchdog Timer
[   15.035812] rk_gmac-dwmac ff540000.ethernet end0: renamed from eth0
[   15.179382] rk_gmac-dwmac ff540000.ethernet end0: Register MEM_TYPE_PAGE_POOL RxQ-0
[   15.243032] rk_gmac-dwmac ff540000.ethernet end0: PHY [stmmac-0:00] driver [YT8531 Gigabit Ethernet] (irq=POLL)
[   15.253610] rk_gmac-dwmac ff540000.ethernet end0: No Safety Features support found
[   15.254333] rk_gmac-dwmac ff540000.ethernet end0: PTP not supported by HW
[   15.255482] rk_gmac-dwmac ff540000.ethernet end0: configuring for phy/rgmii link mode
[   18.334667] rk_gmac-dwmac ff540000.ethernet end0: Link is Up - 1Gbps/Full - flow control off

and gets an IPv4 address from DHCP. Within a few seconds the connection just gets slower and slower, down to a few bits a second. I have tried different cables, as well as multiple routers, as well as crossover to rule out any external hardware issue. I did notice when running wireshard that it appears to be sending out a lot of TCP retransmissions when this is occurring.

I have tried all 4 of the images, and they all have the same issue. I have also noticed I can't seem to get the second NIC to be detected by the OS or powered on at all. I also was able to validate with a USB Wireless Cellular connection that it is able to connect to the internet and download things fine (using a serial connection since eventually SSH drops off)

I suspect it may be a u-boot problem since it happens in all of the images, but am not sure how to debug any potential power-up/regulator issues that might be occurring.

Here's my output from lsmod on trixie:

Module                  Size  Used by
sd_mod                 65536  0
t10_pi                 16384  1 sd_mod
crc64_rocksoft_generic    12288  1
crc64_rocksoft         16384  1 t10_pi
crc_t10dif             16384  1 t10_pi
crct10dif_generic      12288  0
crc64                  16384  2 crc64_rocksoft,crc64_rocksoft_generic
sg                     32768  0
ofpart                 16384  0
spi_nor               102400  0
evdev                  20480  1
mtd                    69632  3 spi_nor,ofpart
motorcomm              24576  1
polyval_ce             12288  0
polyval_generic        12288  1 polyval_ce
dwmac_rk               28672  0
stmmac_platform        20480  1 dwmac_rk
ghash_ce               16384  0
gf128mul               12288  2 polyval_generic,ghash_ce
sha2_ce                12288  0
stmmac                241664  3 stmmac_platform,dwmac_rk
uas                    24576  0
hantro_vpu            262144  0
usb_storage            73728  1 uas
sha256_arm64           24576  1 sha2_ce
pcs_xpcs               16384  1 stmmac
rockchip_vdec          40960  0
rndis_host             20480  0
des_generic            12288  0
phylink                49152  2 stmmac,pcs_xpcs
sha1_ce                12288  0
v4l2_vp9               20480  2 rockchip_vdec,hantro_vpu
ecb                    12288  0
cdc_ether              16384  1 rndis_host
scsi_mod              233472  4 sd_mod,usb_storage,uas,sg
v4l2_h264              16384  2 rockchip_vdec,hantro_vpu
of_mdio                16384  2 stmmac_platform,stmmac
usbnet                 40960  2 rndis_host,cdc_ether
fixed_phy              12288  1 of_mdio
videobuf2_dma_contig    20480  2 rockchip_vdec,hantro_vpu
mii                    16384  1 usbnet
scsi_common            12288  5 scsi_mod,sd_mod,usb_storage,uas,sg
fwnode_mdio            16384  1 of_mdio
aes_neon_bs            24576  0
v4l2_mem2mem           32768  2 rockchip_vdec,hantro_vpu
libphy                114688  7 fwnode_mdio,of_mdio,stmmac,fixed_phy,motorcomm,phylink,pcs_xpcs
leds_gpio              12288  0
videobuf2_memops       16384  1 videobuf2_dma_contig
videobuf2_v4l2         20480  3 rockchip_vdec,hantro_vpu,v4l2_mem2mem
aes_neon_blk           24576  1 aes_neon_bs
lima                   65536  0
videodev              245760  4 rockchip_vdec,videobuf2_v4l2,hantro_vpu,v4l2_mem2mem
aes_ce_blk             28672  0
aes_ce_cipher          12288  1 aes_ce_blk
videobuf2_common       53248  6 rockchip_vdec,videobuf2_dma_contig,videobuf2_v4l2,hantro_vpu,v4l2_mem2mem,videobuf2_memops
gpu_sched              45056  1 lima
drm_shmem_helper       16384  1 lima
mc                     57344  6 rockchip_vdec,videodev,videobuf2_v4l2,hantro_vpu,videobuf2_common,v4l2_mem2mem
rk805_pwrkey           12288  0
syscon_reboot_mode     12288  0
nvmem_rockchip_efuse    12288  0
reboot_mode            12288  1 syscon_reboot_mode
dw_wdt                 12288  0
rockchip_thermal       24576  0
rk_crypto              28672  0
pl330                  28672  2
crypto_engine          16384  1 rk_crypto
libdes                 20480  2 des_generic,rk_crypto
cfg80211              831488  0
cpufreq_dt             16384  0
rfkill                 32768  2 cfg80211
fuse                  135168  1
loop                   32768  0
drm                   466944  4 gpu_sched,lima,drm_shmem_helper
efi_pstore             12288  0
dm_mod                143360  0
dax                    32768  1 dm_mod
configfs               49152  1
nfnetlink              16384  2
ip_tables              24576  0
x_tables               32768  1 ip_tables
autofs4                45056  2
ext4                  782336  1
crc16                  12288  1 ext4
mbcache                16384  1 ext4
jbd2                  135168  1 ext4
crc32c_generic         12288  2
xhci_plat_hcd          16384  0
xhci_hcd              249856  1 xhci_plat_hcd
rk808_regulator        49152  12
gpio_rockchip          20480  7
ohci_platform          12288  0
dwc2                  241664  0
ohci_hcd               53248  1 ohci_platform
ehci_platform          16384  0
ehci_hcd               86016  1 ehci_platform
dwc3                  163840  0
crct10dif_ce           12288  1
crct10dif_common       12288  3 crct10dif_ce,crct10dif_generic,crc_t10dif
usbcore               270336  12 ohci_platform,xhci_hcd,ohci_hcd,ehci_platform,usbnet,dwc2,usb_storage,rndis_host,ehci_hcd,cdc_ether,uas,xhci_plat_hcd
udc_core               53248  2 dwc2,dwc3
dw_mmc_rockchip        16384  0
fixed                  16384  7
pinctrl_rockchip      114688  52
dw_mmc_pltfm           12288  1 dw_mmc_rockchip
roles                  12288  2 dwc2,dwc3
phy_rockchip_inno_usb2    40960  6
ulpi                   16384  1 dwc3
usb_common             12288  11 ohci_platform,xhci_hcd,ohci_hcd,ehci_platform,phy_rockchip_inno_usb2,dwc2,usbcore,ehci_hcd,xhci_plat_hcd,dwc3,udc_core
dw_mmc                 45056  2 dw_mmc_pltfm,dw_mmc_rockchip
gpio_syscon            16384  0
pwm_rockchip           12288  0
io_domain              20480  0
spi_rockchip           16384  0
i2c_rk3x               20480  0

Here's also my output from ifconfig:

        inet 192.168.88.242  netmask 255.255.255.0  broadcast 192.168.88.255
        inet6 fe80::84e0:c0ff:feea:faa9  prefixlen 64  scopeid 0x20<link>
        ether 86:e0:c0:ea:fa:a9  txqueuelen 1000  (Ethernet)
        RX packets 1744  bytes 109074 (106.5 KiB)
        RX errors 0  dropped 1703  overruns 0  frame 0
        TX packets 66  bytes 8163 (7.9 KiB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
        device interrupt 56  

which has that entry for device interrupt at the end which looks suspicious, but it may be a symptom rather than the cause.

johang commented 10 months ago

Do you have packet loss? You can confirm this with ping. Try forcing 100 Mbps and 10 Mbps and see if that improves it. Sounds like a HW issue to be honest.

DaveWK commented 10 months ago

Hi, I confirmed with ping and wireshark that I am getting packet loss when running these sd-images. I have tried a few different cords, and switches to confirm they were in working order. Each time when I get into a root shell and run apt update it will have stalled out in a few seconds due to retransmission and packet loss.

But in counter-point to it being a hardware issue, when I test using the latest Armbian Trixie rolling release (6.6.10). I am able to finish an initial apt update successfully before the network connection slows down substantially.

I noticed this merge in arbian commit that seems to give the nics some extra time to power up: https://github.com/armbian/build/commit/ac9373aca47fe6967827503bd32eb5d66b8a7128

I suspect that the issue may be the issue?

johang commented 10 months ago

https://lore.kernel.org/linux-rockchip/20231216040723.17864-1-cnsztl@gmail.com/T/#u

Possibly related.

DaveWK commented 10 months ago

Interesting -- I will give that a shot to see if that improves anything, but my current armbian dts isn't using that.. here's my live one:

 ethernet@ff540000 {
                compatible = "rockchip,rk3328-gmac";
                reg = <0x00 0xff540000 0x00 0x10000>;
                interrupts = <0x00 0x18 0x04>;
                interrupt-names = "macirq";
                clocks = <0x02 0x64 0x02 0x57 0x02 0x58 0x02 0x5a 0x02 0x59 0x02 0x96 0x02 0xdf>;
                clock-names = "stmmaceth\0mac_clk_rx\0mac_clk_tx\0clk_mac_ref\0clk_mac_refout\0aclk_mac\0pclk_mac";
                resets = <0x02 0x63>;
                reset-names = "stmmaceth";
                rockchip,grf = <0x3d>;
                snps,txpbl = <0x04>;
                status = "okay";
                assigned-clocks = <0x02 0x64 0x02 0x66>;
                assigned-clock-parents = <0x54 0x54>;
                clock_in_out = "input";
                phy-mode = "rgmii";
                phy-supply = <0x1e>;
                pinctrl-0 = <0x55>;
                pinctrl-names = "default";
                snps,aal;
                phy-handle = <0x56>;
                tx_delay = <0x19>;
                rx_delay = <0x05>;
                phandle = <0xa1>;

                mdio {
                        compatible = "snps,dwmac-mdio";
                        #address-cells = <0x01>;
                        #size-cells = <0x00>;
ethernet-phy@0 {
                                compatible = "ethernet-phy-id4f51.e91b\0ethernet-phy-ieee802.3-c22";
                                reg = <0x00>;
                                motorcomm,clk-out-frequency-hz = <0x7735940>;
                                motorcomm,keep-pll-enabled;
                                motorcomm,auto-sleep-disabled;
                                pinctrl-0 = <0x57>;
                                pinctrl-names = "default";
                                reset-assert-us = <0x3a98>;
                                reset-deassert-us = <0xc350>;
                                reset-gpios = <0x2b 0x12 0x01>;
                                phandle = <0x56>;
                        };
               };
        };