search

open-switch / opx-cps

https://openswitch.net
6 stars 15 forks source link

Server (untagged) unable to ping SVI gateway hosted of ToR #80

Closed Ragsboss closed 6 years ago

Ragsboss commented 6 years ago

I have a leaf-spine network, where a VLAN is created on the leaf (aka ToR) and server facing interfaces are added as members to the bridge corresponding to this VLAN. An IP address is configured on this bridge to represent the SVI. On server, IP is configured on a physical interface and sending untagged packets to ToR SVI IP is not working. Below are further details

OPX version is

root@rack-1-leaf-1:~# opx-show-version 
OS_NAME="OPX"
OS_VERSION="2.2.1"
PLATFORM="S4048-ON"
ARCHITECTURE="x86_64"
INTERNAL_BUILD_ID="OpenSwitch blueprint for Dell 1.0.0"
BUILD_VERSION="2.2.1.0"
BUILD_DATE="2018-02-13T13:48:47-0800"
INSTALL_DATE="2018-02-23T03:31:30+0000"
SYSTEM_UPTIME= 59 minutes
SYSTEM_STATE= running
UPGRADED_PACKAGES=no
ALTERED_PACKAGES=no

Contents of /etc/network/interfaces file are

root@rack-1-leaf-1:~# cat /etc/network/interfaces
auto eth0
  iface eth0 inet static
  address 172.16.53.77
  netmask 255.255.254.0
  gateway 172.16.53.254
  broadcast 172.16.53.255
  dns-nameservers 8.8.8.8
auto lo
iface lo inet loopback
# BEGIN ANSIBLE MANAGED BLOCK
post-up ip netns add mgmt; sleep 1
post-up ln -s /proc/1/ns/net /var/run/netns/default
post-up ip link set eth0 netns mgmt; sleep 1

post-up ip netns exec mgmt ip link set dev eth0 up
post-up ip netns exec mgmt ip addr add 172.16.53.77/23 dev eth0 broadcast 172.16.53.255
post-up ip netns exec mgmt ip route add default dev eth0 via 172.16.53.254
post-up ip netns exec mgmt /usr/sbin/sshd -o PidFile=/run/sshd-mgmt.pid
# END ANSIBLE MANAGED BLOCK
auto lo
iface lo inet loopback

# This file was generated by AOS. Do not edit by hand.
#
# The loopback interface
auto lo
iface lo inet loopback
  address 172.16.0.0/32

# Fabric interfaces
auto e101-049-0
iface e101-049-0
    address 172.16.0.19/31
    alias facing_spine2:e101-022-0
    mtu 9050

auto e101-053-0
iface e101-053-0
    address 172.16.0.7/31
    alias facing_spine1:e101-012-0
    mtu 9050

# L3Edge interfaces
auto e101-006-0
iface e101-006-0
    address 172.16.0.32/31
    alias facing_sio-router-a

# Unused interfaces
auto e101-015-0
iface e101-015-0

auto e101-016-0
iface e101-016-0

auto e101-017-0
iface e101-017-0

auto e101-018-0
iface e101-018-0

auto e101-019-0
iface e101-019-0

auto e101-020-0
iface e101-020-0

auto e101-025-0
iface e101-025-0

auto e101-028-0
iface e101-028-0

auto e101-029-0
iface e101-029-0

auto e101-039-0
iface e101-039-0

auto e101-047-0
iface e101-047-0

auto e101-048-0
iface e101-048-0

auto e101-050-0
iface e101-050-0

auto e101-051-0
iface e101-051-0

auto e101-052-0
iface e101-052-0

auto e101-054-0
iface e101-054-0

auto e101-001-0
iface e101-001-0
    alias to.rack-1-sds1:enp1s0f0
auto e101-002-0
iface e101-002-0
    alias to.rack-1-sds9:enp1s0f0
auto e101-003-0
iface e101-003-0
    alias to.rack-1-sds32:enp1s0f0
auto e101-004-0
iface e101-004-0
    alias to.rack-1-sds33:enp1s0f0
auto e101-005-0
iface e101-005-0
    alias to.rack-1-sds34:enp1s0f0
auto e101-007-0
iface e101-007-0
    alias to.rack-1-sds22:enp1s0f0
auto e101-008-0
iface e101-008-0
    alias to.rack-1-sds20:enp1s0f0
auto e101-009-0
iface e101-009-0
    alias to.rack-1-sds2:enp1s0f1
auto e101-010-0
iface e101-010-0
    alias to.rack-1-sds23:enp1s0f0
auto e101-011-0
iface e101-011-0
    alias to.rack-1-sds19:enp1s0f0
auto e101-012-0
iface e101-012-0
    alias to.rack-1-sds21:enp1s0f0
auto e101-013-0
iface e101-013-0
    alias to.rack-1-sds18:enp1s0f0
auto e101-014-0
iface e101-014-0
    alias to.rack-1-sds17:enp1s0f0
auto e101-021-0
iface e101-021-0
    alias to.rack-1-sds5:enp1s0f0
auto e101-022-0
iface e101-022-0
    alias to.rack-1-sds8:enp1s0f0
auto e101-023-0
iface e101-023-0
    alias to.rack-1-sds3:enp1s0f0
auto e101-024-0
iface e101-024-0
    alias to.rack-1-sds4:enp1s0f0
auto e101-026-0
iface e101-026-0
    alias to.rack-1-sds6:enp1s0f0
auto e101-027-0
iface e101-027-0
    alias to.rack-1-sds7:enp1s0f0
auto e101-030-0
iface e101-030-0
    alias to.rack-1-mdm1:enp1s0f0
auto e101-031-0
iface e101-031-0
    alias to.rack-1-sds16:enp1s0f0
auto e101-032-0
iface e101-032-0
    alias to.rack-1-sds15:enp1s0f0
auto e101-033-0
iface e101-033-0
    alias to.rack-1-sds13:enp1s0f0
auto e101-034-0
iface e101-034-0
    alias to.rack-1-sds14:enp1s0f0
auto e101-035-0
iface e101-035-0
    alias to.rack-1-sds12:enp1s0f0
auto e101-036-0
iface e101-036-0
    alias to.rack-1-sds10:enp1s0f0
auto e101-037-0
iface e101-037-0
    alias to.rack-1-sds24:enp1s0f0
auto e101-038-0
iface e101-038-0
    alias to.rack-1-sds11:enp1s0f0
auto e101-040-0
iface e101-040-0
    alias to.rack-1-sds29:enp1s0f0
auto e101-041-0
iface e101-041-0
    alias to.rack-1-sds26:enp1s0f0
auto e101-042-0
iface e101-042-0
    alias to.rack-1-sds31:enp1s0f0
auto e101-043-0
iface e101-043-0
    alias to.rack-1-sds28:enp1s0f0
auto e101-044-0
iface e101-044-0
    alias to.rack-1-sds25:enp1s0f0
auto e101-045-0
iface e101-045-0
    alias to.rack-1-sds30:enp1s0f0
auto e101-046-0
iface e101-046-0
    alias to.rack-1-sds27:enp1s0f0
auto br500
iface br500
    address 12.12.10.1/26
    bridge_ports e101-024-0.500 e101-044-0.500 e101-001-0 e101-002-0 e101-003-0 e101-004-0 e101-005-0 e101-007-0 e101-008-0 e101-009-0 e101-010-0 e101-011-0 e101-012-0 e101-013-0 e101-014-0 e101-021-0 e101-022-0 e101-023-0 e101-026-0 e101-027-0 e101-030-0 e101-031-0 e101-032-0 e101-033-0 e101-034-0 e101-035-0 e101-036-0 e101-037-0 e101-038-0 e101-040-0 e101-041-0 e101-042-0 e101-043-0 e101-045-0 e101-046-0
    bridge_stp on
    bridge_bridgeprio 8192

Output of ip addr on ToR

root@rack-1-leaf-1:~# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 172.16.0.0/32 scope global lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
5: npu-0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether f2:3b:80:b7:4b:e6 brd ff:ff:ff:ff:ff:ff
6: e101-009-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:cd brd ff:ff:ff:ff:ff:ff
7: e101-010-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:ce brd ff:ff:ff:ff:ff:ff
8: e101-011-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:cf brd ff:ff:ff:ff:ff:ff
9: e101-012-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:d0 brd ff:ff:ff:ff:ff:ff
10: e101-013-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:d1 brd ff:ff:ff:ff:ff:ff
11: e101-014-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:d2 brd ff:ff:ff:ff:ff:ff
12: e101-015-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:d3 brd ff:ff:ff:ff:ff:ff
13: e101-016-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:d4 brd ff:ff:ff:ff:ff:ff
14: e101-017-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:d5 brd ff:ff:ff:ff:ff:ff
15: e101-018-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:d6 brd ff:ff:ff:ff:ff:ff
16: e101-019-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:d7 brd ff:ff:ff:ff:ff:ff
17: e101-020-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:d8 brd ff:ff:ff:ff:ff:ff
18: e101-021-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:d9 brd ff:ff:ff:ff:ff:ff
19: e101-022-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:da brd ff:ff:ff:ff:ff:ff
20: e101-023-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:db brd ff:ff:ff:ff:ff:ff
21: e101-024-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:dc brd ff:ff:ff:ff:ff:ff
22: e101-001-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq master br500 state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:c5 brd ff:ff:ff:ff:ff:ff
23: e101-002-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq master br500 state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:c6 brd ff:ff:ff:ff:ff:ff
24: e101-003-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq master br500 state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:c7 brd ff:ff:ff:ff:ff:ff
25: e101-004-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq master br500 state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:c8 brd ff:ff:ff:ff:ff:ff
26: e101-005-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:c9 brd ff:ff:ff:ff:ff:ff
27: e101-006-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:ca brd ff:ff:ff:ff:ff:ff
    inet 172.16.0.32/31 scope global e101-006-0
       valid_lft forever preferred_lft forever
    inet6 fe80::3617:ebff:fef4:65ca/64 scope link 
       valid_lft forever preferred_lft forever
28: e101-007-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:cb brd ff:ff:ff:ff:ff:ff
29: e101-008-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:cc brd ff:ff:ff:ff:ff:ff
30: e101-025-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:dd brd ff:ff:ff:ff:ff:ff
31: e101-026-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:de brd ff:ff:ff:ff:ff:ff
32: e101-027-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:df brd ff:ff:ff:ff:ff:ff
33: e101-028-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:e0 brd ff:ff:ff:ff:ff:ff
34: e101-029-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:e1 brd ff:ff:ff:ff:ff:ff
35: e101-030-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:e2 brd ff:ff:ff:ff:ff:ff
36: e101-031-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:e3 brd ff:ff:ff:ff:ff:ff
37: e101-032-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:e4 brd ff:ff:ff:ff:ff:ff
38: e101-033-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:e5 brd ff:ff:ff:ff:ff:ff
39: e101-034-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:e6 brd ff:ff:ff:ff:ff:ff
40: e101-035-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:e7 brd ff:ff:ff:ff:ff:ff
41: e101-036-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:e8 brd ff:ff:ff:ff:ff:ff
42: e101-037-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:e9 brd ff:ff:ff:ff:ff:ff
43: e101-038-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:ea brd ff:ff:ff:ff:ff:ff
44: e101-039-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:eb brd ff:ff:ff:ff:ff:ff
45: e101-040-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:ec brd ff:ff:ff:ff:ff:ff
46: e101-041-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:ed brd ff:ff:ff:ff:ff:ff
47: e101-042-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:ee brd ff:ff:ff:ff:ff:ff
48: e101-043-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:ef brd ff:ff:ff:ff:ff:ff
49: e101-044-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:f0 brd ff:ff:ff:ff:ff:ff
50: e101-045-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:f1 brd ff:ff:ff:ff:ff:ff
51: e101-046-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master br500 state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:f2 brd ff:ff:ff:ff:ff:ff
52: e101-047-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:f3 brd ff:ff:ff:ff:ff:ff
53: e101-048-0: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN group default qlen 1000
    link/ether 34:17:eb:f4:65:f4 brd ff:ff:ff:ff:ff:ff
54: e101-054-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 34:17:eb:f4:66:09 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::3617:ebff:fef4:6609/64 scope link 
       valid_lft forever preferred_lft forever
55: e101-053-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9050 qdisc mq state UP group default qlen 1000
    link/ether 34:17:eb:f4:66:05 brd ff:ff:ff:ff:ff:ff
    inet 172.16.0.7/31 scope global e101-053-0
       valid_lft forever preferred_lft forever
    inet6 fe80::3617:ebff:fef4:6605/64 scope link 
       valid_lft forever preferred_lft forever
56: e101-050-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:f9 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::3617:ebff:fef4:65f9/64 scope link 
       valid_lft forever preferred_lft forever
57: e101-049-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 9050 qdisc mq state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:f5 brd ff:ff:ff:ff:ff:ff
    inet 172.16.0.19/31 scope global e101-049-0
       valid_lft forever preferred_lft forever
    inet6 fe80::3617:ebff:fef4:65f5/64 scope link 
       valid_lft forever preferred_lft forever
58: e101-052-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 34:17:eb:f4:66:01 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::3617:ebff:fef4:6601/64 scope link 
       valid_lft forever preferred_lft forever
59: e101-051-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 34:17:eb:f4:65:fd brd ff:ff:ff:ff:ff:ff
    inet6 fe80::3617:ebff:fef4:65fd/64 scope link 
       valid_lft forever preferred_lft forever
60: br500: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default 
    link/ether 34:17:eb:f4:66:8d brd ff:ff:ff:ff:ff:ff
    inet 12.12.10.1/26 scope global br500
       valid_lft forever preferred_lft forever
    inet6 fe80::3617:ebff:fef4:65c5/64 scope link 
       valid_lft forever preferred_lft forever
61: e101-044-0.500@e101-044-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br500 state UP group default 
    link/ether 34:17:eb:f4:65:f0 brd ff:ff:ff:ff:ff:ff
62: e101-024-0.500@e101-024-0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master br500 state UP group default 
    link/ether 34:17:eb:f4:65:dc brd ff:ff:ff:ff:ff:ff

Output of brctl show

root@rack-1-leaf-1:~# brctl show
bridge name bridge id       STP enabled interfaces
br500       2000.3417ebf4668d   yes     e101-001-0
                            e101-002-0
                            e101-003-0
                            e101-004-0
                            e101-005-0
                            e101-007-0
                            e101-008-0
                            e101-009-0
                            e101-010-0
                            e101-011-0
                            e101-012-0
                            e101-013-0
                            e101-014-0
                            e101-021-0
                            e101-022-0
                            e101-023-0
                            e101-024-0
                            e101-024-0.500
                            e101-026-0
                            e101-027-0
                            e101-030-0
                            e101-031-0
                            e101-032-0
                            e101-033-0
                            e101-034-0
                            e101-035-0
                            e101-036-0
                            e101-037-0
                            e101-038-0
                            e101-040-0
                            e101-041-0
                            e101-042-0
                            e101-043-0
                            e101-044-0.500
                            e101-045-0
                            e101-046-0

Output of opx-switch-shell 'vlan show'

root@rack-1-leaf-1:~# opx-switch-shell "vlan show"
vlan 1  ports none (0x0000000000000000000000000000000000000000000000000000000000000000), untagged none (0x0000000000000000000000000000000000000000000000000000000000000000) MCAST_FLOOD_UNKNOWN
vlan 500    ports xe0-xe4,xe6-xe13,xe20-xe23,xe25-xe26,xe29-xe37,xe39-xe45 (0x00000000000000000000000000000000000000000000000000007f7fcde07fbe), untagged xe0-xe4,xe6-xe13,xe20-xe23,xe25-xe26,xe29-xe37,xe39-xe42,xe44-xe45 (0x00000000000000000000000000000000000000000000000000006f7fcde07fbe) MCAST_FLOOD_UNKNOWN
vlan 4095   ports xe (0x0000000000000000000000000000000000000000000001fffffffffffffffffe), untagged xe (0x0000000000000000000000000000000000000000000001fffffffffffffffffe) MCAST_FLOOD_UNKNOWN
SAI.0>

Output of cps_config_vlan.py --show on ToR

root@rack-1-leaf-1:~# cps_config_vlan.py --show
Key: 1.19.44.2883617.2883612.2883613.
dell-base-if-cmn/if/interfaces/interface/if-index = 62
dell-if/if/interfaces/interface/phys-address = 34:17:eb:f4:66:8d
dell-if/if/interfaces/interface/untagged-ports = e101-046-0,e101-007-0,e101-035-0,e101-021-0,e101-036-0,e101-011-0,e101-012-0,e101-037-0,e101-008-0,e101-009-0,e101-034-0,e101-003-0,e101-004-0,e101-005-0,e101-032-0,e101-013-0,e101-045-0,e101-022-0,e101-027-0,e101-014-0,e101-010-0,e101-030-0,e101-023-0,e101-040-0,e101-033-0,e101-041-0,e101-031-0,e101-042-0,e101-001-0,e101-043-0,e101-002-0,e101-026-0,e101-038-0
base-if-vlan/if/interfaces/interface/id = 500
if/interfaces/interface/name = br500
dell-if/if/interfaces/interface/vlan-type = 0
dell-if/if/interfaces/interface/learning-mode = 1
dell-if/if/interfaces/interface/mtu = 1532
if/interfaces/interface/enabled = 0
dell-if/if/interfaces/interface/tagged-ports = e101-024-0,e101-044-0

Output of ip addr on server

root@rack-1-SDS4:/var/log/aos# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 10.0.1.4/32 scope global lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eno1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 20:47:47:7e:57:34 brd ff:ff:ff:ff:ff:ff
    inet 172.16.52.113/23 brd 172.16.53.255 scope global eno1
       valid_lft forever preferred_lft forever
    inet6 fe80::2247:47ff:fe7e:5734/64 scope link 
       valid_lft forever preferred_lft forever
3: eno2: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN group default qlen 1000
    link/ether 20:47:47:7e:57:36 brd ff:ff:ff:ff:ff:ff
4: enp1s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq portid 000e1e986ce0 state UP group default qlen 1000
    link/ether 00:0e:1e:98:6c:e0 brd ff:ff:ff:ff:ff:ff
    inet 12.12.10.6/26 scope global enp1s0f0
       valid_lft forever preferred_lft forever
5: enp1s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq portid 000e1e986ce2 state UP group default qlen 1000
    link/ether 00:0e:1e:98:6c:e2 brd ff:ff:ff:ff:ff:ff
    inet 12.12.10.70/26 scope global enp1s0f1
       valid_lft forever preferred_lft forever

Ping from server to ToR SVI IP

root@rack-1-SDS4:/var/log/aos# ping 12.12.10.1
PING 12.12.10.1 (12.12.10.1) 56(84) bytes of data.
^C
--- 12.12.10.1 ping statistics ---
852 packets transmitted, 0 received, 100% packet loss, time 850999ms

tcpdump from server interface

root@rack-1-SDS4:/var/log/aos# tcpdump -i enp1s0f0
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on enp1s0f0, link-type EN10MB (Ethernet), capture size 262144 bytes
13:27:01.539486 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 43
13:27:01.539494 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 43
13:27:03.539387 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 43
13:27:03.539396 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 43
13:27:05.539401 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 43
13:27:05.539409 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 43
13:27:07.475113 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 1, length 64
13:27:07.539487 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 43
13:27:07.539512 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 43
13:27:08.474889 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 2, length 64
13:27:09.289365 LLDP, length 260: Rack-1-Leaf-1
13:27:09.474828 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 3, length 64
13:27:09.539262 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 43
13:27:09.539275 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 43
13:27:10.474864 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 4, length 64
13:27:11.474837 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 5, length 64
13:27:11.539024 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 43
13:27:11.539033 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 43
13:27:12.474826 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 6, length 64
13:27:13.474777 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 7, length 64
13:27:13.539049 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 43
13:27:13.539059 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 43
13:27:14.474849 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 8, length 64
13:27:15.474846 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 10519, seq 9, length 64

tcpdump from ToR interface

root@rack-1-leaf-1:~# tcpdump -i e101-024-0
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on e101-024-0, link-type EN10MB (Ethernet), capture size 262144 bytes
20:30:21.996393 LLDP, length 260: Rack-1-Leaf-1
20:30:22.250998 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:22.251025 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:22.700242 LLDP, length 85
20:30:24.250975 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:24.251000 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:26.250975 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:26.251000 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:28.250978 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:28.251018 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:30.250977 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:30.251013 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:32.250980 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:32.251015 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:34.251008 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:34.251034 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:35.298740 LLDP, length 216: rack-1-SDS4
20:30:36.250976 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:36.251002 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:38.250974 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:38.251001 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:40.250976 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:40.251014 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
20:30:42.250974 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8024, length 35
20:30:42.251006 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8002, length 35
Ragsboss commented 6 years ago

Tagged traffic from server does work fine.

ip link add link enp1s0f0 name enp1s0f0.500 type vlan id 500
ip link set enp1s0f0.500 up
ip addr add 12.12.10.6/26 dev enp1s0f0.500
ping 12.12.10.1
PING 12.12.10.1 (12.12.10.1) 56(84) bytes of data.
64 bytes from 12.12.10.1: icmp_seq=1 ttl=64 time=0.273 ms
64 bytes from 12.12.10.1: icmp_seq=2 ttl=64 time=0.406 ms

tcpdump from ToR interface

root@rack-1-leaf-1:~# tcpdump -i e101-024-0
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on e101-024-0, link-type EN10MB (Ethernet), capture size 262144 bytes
21:08:06.772738 LLDP, length 260: Rack-1-Leaf-1
21:08:07.243321 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8010, length 35
21:08:09.243331 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8010, length 35
21:08:11.243314 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8010, length 35
21:08:12.391968 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 11487, seq 1, length 64
21:08:12.392034 IP 12.12.10.1 > 12.12.10.6: ICMP echo reply, id 11487, seq 1, length 64
21:08:13.243310 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8010, length 35
21:08:13.391218 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 11487, seq 2, length 64
21:08:13.391287 IP 12.12.10.1 > 12.12.10.6: ICMP echo reply, id 11487, seq 2, length 64
21:08:14.391237 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 11487, seq 3, length 64
21:08:14.391300 IP 12.12.10.1 > 12.12.10.6: ICMP echo reply, id 11487, seq 3, length 64
21:08:15.243325 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8010, length 35
21:08:15.391284 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 11487, seq 4, length 64
21:08:15.391347 IP 12.12.10.1 > 12.12.10.6: ICMP echo reply, id 11487, seq 4, length 64
21:08:16.391283 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 11487, seq 5, length 64
21:08:16.391342 IP 12.12.10.1 > 12.12.10.6: ICMP echo reply, id 11487, seq 5, length 64
21:08:17.243316 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8010, length 35
21:08:17.391326 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 11487, seq 6, length 64
21:08:17.391396 IP 12.12.10.1 > 12.12.10.6: ICMP echo reply, id 11487, seq 6, length 64
21:08:18.391332 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 11487, seq 7, length 64
21:08:18.391390 IP 12.12.10.1 > 12.12.10.6: ICMP echo reply, id 11487, seq 7, length 64
21:08:19.243318 STP 802.1d, Config, Flags [none], bridge-id 2000.34:17:eb:f4:66:8d.8010, length 35
21:08:19.391376 IP 12.12.10.6 > 12.12.10.1: ICMP echo request, id 11487, seq 8, length 64
21:08:19.391443 IP 12.12.10.1 > 12.12.10.6: ICMP echo reply, id 11487, seq 8, length 64
madhu222 commented 6 years ago

@Ragsboss You have to configure both tagged and untagged port on the server side also. I have one server which is connected to the switch. Below are the configurations I did.

Server Side

auto br500 iface br500 inet static address 12.12.10.1/26 bridge_ports ens192 ens192.500 bridge_stp on bridge_bridgeprio 8192

root@ubuntu-opx1:~# brctl show bridge name bridge id STP enabled interfaces br500 2000.000c29e2e790 yes ens192 ens192.500 `

Switch Side

Using CPS config , first add the tagged port to the bridge.

cps_config_vlan.py --add --id 500 --vlantype 1 --port e101-027-1 --tagged

Add the untagged port to the bridge next.

cps_config_vlan.py --addport --name br500 --port e101-027-1

root@AG1:~# ifconfig br500 br500 Link encap:Ethernet HWaddr f4:8e:38:1c:0c:56 inet addr:12.12.10.6 Bcast:12.12.10.63 Mask:255.255.255.192 UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1 RX packets:356 errors:0 dropped:0 overruns:0 frame:0 TX packets:61 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:19024 (18.5 KiB) TX bytes:5158 (5.0 KiB)

root@AG1:~# brctl show br500 bridge name bridge id STP enabled interfaces br500 8000.f48e381c0c56 no e101-027-1 e101-027-1.500

root@AG1:~# opx-switch-shell 'vlan show 500' vlan 500 ports xe56 (0x0000000000000000000000000000000000000000200000000000000000000000), untagged xe56(0x00000000000000000000000000000000000000200000000000000000000000)MCAST_FLOOD_UNKNOWN `

root@AG1:~# cps_config_vlan.py --show --name br500 Key: 1.19.44.2883617.2883612.2883613. dell-base-if-cmn/if/interfaces/interface/if-index = 291 dell-if/if/interfaces/interface/phys-address = f4:8e:38:1c:0c:56 base-if-vlan/if/interfaces/interface/id = 500 dell-if/if/interfaces/interface/untagged-ports = e101-027-1 if/interfaces/interface/type = ianaift:l2vlan if/interfaces/interface/name = br500 dell-if/if/interfaces/interface/vlan-type = 1 dell-if/if/interfaces/interface/learning-mode = 1 dell-if/if/interfaces/interface/mtu = 1532 if/interfaces/interface/enabled = 0 dell-if/if/interfaces/interface/tagged-ports = e101-027-1

root@AG1:~# ifconfig br500 br500 Link encap:Ethernet HWaddr f4:8e:38:1c:0c:56 inet addr:12.12.10.6 Bcast:12.12.10.63 Mask:255.255.255.192 UP BROADCAST RUNNING PROMISC MULTICAST MTU:1500 Metric:1 RX packets:395 errors:0 dropped:0 overruns:0 frame:0 TX packets:61 errors:0 dropped:0 overruns:0 carrier:0 collisions:0 txqueuelen:0 RX bytes:20974 (20.4 KiB) TX bytes:5158 (5.0 KiB)

root@AG1:~# ping 12.12.10.1 PING 12.12.10.1 (12.12.10.1) 56(84) bytes of data. 64 bytes from 12.12.10.1: icmp_seq=1 ttl=64 time=2003 ms 64 bytes from 12.12.10.1: icmp_seq=2 ttl=64 time=994 ms 64 bytes from 12.12.10.1: icmp_seq=3 ttl=64 time=0.260 ms 64 bytes from 12.12.10.1: icmp_seq=4 ttl=64 time=0.319 ms 64 bytes from 12.12.10.1: icmp_seq=5 ttl=64 time=0.254 ms ^C --- 12.12.10.1 ping statistics --- 5 packets transmitted, 5 received, 0% packet loss, time 4006ms rtt min/avg/max/mdev = 0.254/599.700/2003.184/800.442 ms, pipe 2 `

Ragsboss commented 6 years ago

@madhu222 thank you for your response. The use case here is a server that wants to and can send only untagged traffic. Additionally, we can't ask server guys to configure bridge or tagged ports on the server. Server side should be oblivious to ToR model and standard interfaces/bonds with properly configured IPs should just work. Any configurations including (but ideally no) workarounds are possible to be done only on ToR side.

I did notice that use of cps_config_vlan.py does fix the issue without requiring server to have bridge or tagged port. I'll need to test this little more tomorrow and will share my findings. However ideally I'm looking for a solution that involves only /etc/network/interfaces file because that give us a unified way to manage all configurations for interfaces, it's idempotent and complete. By complete, I mean removing bridges from that file and doing an ifreload will add/remove bridges appropriately[1].

Footnotes: [1] Although, I'm noticing bridge removal is not happening upon removal of entries in interfaces - for which I'll file a separate issue after further investigation.

jeff-yin commented 6 years ago

Hi @Ragsboss -- you won't be able to configure an untagged VLAN in /etc/network/interfaces because of a limitation in the kernel. The kernel doesn't generate netlink messages for the Linux CPS Adapter to read and install the VLAN into the NPU, so configuring untagged VLANs in OPX must be done via cps_config_vlan.py.

@madhu222 , would it be possible to add "cps_config_vlan.py" lines to the /etc/network/interfaces file to get the untagged VLANs created?

madhu222 commented 6 years ago

@Ragsboss you can use only untagged on the server side. Example configurations below.

auto br500 iface br500 inet static address 12.12.10.1/26 bridge_ports ens192 bridge_stp on bridge_bridgeprio 8192

On the switch side configurations below.

auto br500 iface br500 inet static address 12.12.10.6/26 bridge_ports e101-027-1 e101-027-1.500 bridge_stp on bridge_bridgeprio 8192

Please let me know if that worked for you.

jeff-yin commented 6 years ago

@Ragsboss does @madhu222 's suggestion work for you?

jeff-yin commented 6 years ago

Closing the issue for now.