PCIe bus never comes alive

arindam-bose commented 2 years ago

I am trying to use LitePCie for my project with the Trenz TE0712 board. The target and platform files are not there in the litex_boards, so I made my own files mainly following the sqrl_acron board. Here is a minimal file example_test.py I created that includes only PCIe in the design. This board uses PCIe_x1 and 64 bit data width.

from litex.build.generic_platform import *
from litex.build.xilinx import XilinxPlatform, VivadoProgrammer
from litex.build.openocd import OpenOCD

import os
import argparse
import sys

from migen import *

# from litex_boards.platforms import acorn

from litex.soc.interconnect.csr import *
from litex.soc.integration.soc_core import *
from litex.soc.integration.builder import *

from litex.soc.cores.clock import *
from litex.soc.cores.led import LedChaser

from litedram.modules import MT41K256M16
from litedram.phy import s7ddrphy

from litepcie.phy.s7pciephy import S7PCIEPHY
from litepcie.core import LitePCIeEndpoint, LitePCIeMSI
from litepcie.frontend.dma import LitePCIeDMA
from litepcie.frontend.wishbone import LitePCIeWishboneBridge
from litepcie.software import generate_litepcie_software

# IOs ----------------------------------------------------------------------------------------------
_io = [
    # clk / rst
    ("clk200", 0,
        Subsignal("p", Pins("H4"), IOStandard("DIFF_SSTL15")),
        Subsignal("n", Pins("G4"), IOStandard("DIFF_SSTL15"))
    ),

    # Leds
    ("sys_led",  0, Pins("U22"), IOStandard("LVCMOS33")),
    ("led2",  0, Pins("W22"), IOStandard("LVCMOS33")),

    # SPIFlash
    ("flash_cs_n", 0, Pins("T19"), IOStandard("LVCMOS33")),
    ("flash", 0,
        Subsignal("mosi", Pins("P22")),
        Subsignal("miso", Pins("R22")),
        Subsignal("wp",   Pins("P21")),
        Subsignal("hold", Pins("R21")),
        IOStandard("LVCMOS33"),
    ),

    # pcie
    ("pcie_x1", 0,
        Subsignal("rst_n", Pins("M17"), IOStandard("LVCMOS33"), Misc("PULLUP=TRUE")),
        Subsignal("clk_p", Pins("F10")),
        Subsignal("clk_n", Pins("E10")),
        Subsignal("rx_p",  Pins("B8")),
        Subsignal("rx_n",  Pins("A8")),
        Subsignal("tx_p",  Pins("B4")),
        Subsignal("tx_n",  Pins("A4")),
    ),

    # dram
    ("ddram", 0,
        Subsignal("a",       Pins(
            "J1 P6 N5 N3 G1 M3 N2 J5",
            "L1 P2 L4 P5 K2 M1 M5"),
            IOStandard("SSTL15")),
        Subsignal("ba",      Pins("P4 H5 H2"), IOStandard("SSTL15")),
        Subsignal("ras_n",   Pins("M6"),  IOStandard("SSTL15")),
        Subsignal("cas_n",   Pins("M2"), IOStandard("SSTL15")),
        Subsignal("we_n",    Pins("J2"),  IOStandard("SSTL15")),
        Subsignal("cs_n",    Pins("K1"), IOStandard("SSTL15")),
        Subsignal("dm",      Pins(
            "W2 Y7 V4 V5"),
            IOStandard("SSTL15")),
        Subsignal("dq",      Pins(
            "T1 U3 U2 U1 Y2 W1 Y1 V2",
            "V7 W9 AB7 AA8 AB8 AB6 Y8 Y9",
            "AB1 AB5 AB3 AA1 Y4 AA5 AB2 W4",
            "T4 U6 T6 AA6 Y6 T5 U5 R6"),
            IOStandard("SSTL15")),
        Subsignal("dqs_p",   Pins("R3 V9 Y3 W6"),
            IOStandard("DIFF_SSTL15")),
        Subsignal("dqs_n",   Pins("R2 V8 AA3 W5"),
            IOStandard("DIFF_SSTL15")),
        Subsignal("clk_p",   Pins("R1"), IOStandard("DIFF_SSTL15")),
        Subsignal("clk_n",   Pins("P1"), IOStandard("DIFF_SSTL15")),
        Subsignal("cke",     Pins("L3"), IOStandard("SSTL15")),
        Subsignal("odt",     Pins("K3"),  IOStandard("SSTL15")),
        Subsignal("reset_n", Pins("H3"),  IOStandard("LVCMOS15")),
        Misc("SLEW=FAST"),
    ),
]

_serial_io = [
    # Serial adapter on P2.
    ("serial", 0,
        Subsignal("tx", Pins("U18")),
        Subsignal("rx", Pins("P16")),
        Misc("SLEW=FAST"),
        IOStandard("LVCMOS33"),
    ),
]

# Platform -----------------------------------------------------------------------------------------
class Platform(XilinxPlatform):
    default_clk_name   = "clk200"
    default_clk_period = 1e9/200e6

    def __init__(self):
        XilinxPlatform.__init__(self, "xc7a200tfbg484-2", _io, toolchain="vivado")
        self.add_platform_command("set_property INTERNAL_VREF 0.750 [get_iobanks 34]")

        self.toolchain.bitstream_commands = [
            "set_property BITSTREAM.CONFIG.SPI_BUSWIDTH 4 [current_design]",
            "set_property BITSTREAM.CONFIG.CONFIGRATE 66 [current_design]",
            "set_property BITSTREAM.GENERAL.COMPRESS TRUE [current_design]",
            "set_property CFGBVS VCCO [current_design]",
            "set_property CONFIG_VOLTAGE 3.3 [current_design]",
        ]

        self.toolchain.additional_commands = [
            # Non-Multiboot SPI-Flash bitstream generation.
            "write_cfgmem -force -format bin -interface spix4 -size 16 -loadbit \"up 0x0 {build_name}.bit\" -file {build_name}.bin",

            # Multiboot SPI-Flash Operational bitstream generation.
            "set_property BITSTREAM.CONFIG.TIMER_CFG 0x0001fbd0 [current_design]",
            "set_property BITSTREAM.CONFIG.CONFIGFALLBACK Enable [current_design]",
            "write_bitstream -force {build_name}_operational.bit ",
            "write_cfgmem -force -format bin -interface spix4 -size 16 -loadbit \"up 0x0 {build_name}_operational.bit\" -file {build_name}_operational.bin",

            # Multiboot SPI-Flash Fallback bitstream generation.
            "set_property BITSTREAM.CONFIG.NEXT_CONFIG_ADDR 0x00400000 [current_design]",
            "write_bitstream -force {build_name}_fallback.bit ",
            "write_cfgmem -force -format bin -interface spix4 -size 16 -loadbit \"up 0x0 {build_name}_fallback.bit\" -file {build_name}_fallback.bin"
        ]

    def create_programmer(self, name='openocd'):
        if name == 'openocd':
            return OpenOCD("openocd_xc7_ft232.cfg", "bscan_spi_xc7a200t.bit")
        elif name == 'vivado':
            # TODO: some board versions may have s25fl128s
            return VivadoProgrammer(flash_part='s25fl256sxxxxxx0-spi-x1_x2_x4')

    def do_finalize(self, fragment):
        XilinxPlatform.do_finalize(self, fragment)
        self.add_period_constraint(self.lookup_request("clk200", loose=True), 1e9/200e6)

# CRG ----------------------------------------------------------------------------------------------

class CRG(Module):
    def __init__(self, platform, sys_clk_freq):
        self.rst = Signal()

        self.clock_domains.cd_sys       = ClockDomain()
        self.clock_domains.cd_sys4x     = ClockDomain(reset_less=True)
        self.clock_domains.cd_sys4x_dqs = ClockDomain(reset_less=True)
        self.clock_domains.cd_idelay    = ClockDomain()

        # Clk/Rst
        clk200 = platform.request("clk200")

        # PLL
        self.submodules.pll = pll = S7PLL()
        self.comb += pll.reset.eq(self.rst)
        pll.register_clkin(clk200, 200e6)
        pll.create_clkout(self.cd_sys,       sys_clk_freq)
        pll.create_clkout(self.cd_sys4x,     4*sys_clk_freq)
        pll.create_clkout(self.cd_sys4x_dqs, 4*sys_clk_freq, phase=90)
        pll.create_clkout(self.cd_idelay,    200e6)

        self.submodules.idelayctrl = S7IDELAYCTRL(self.cd_idelay)

# PCIeSoC -----------------------------------------------------------------------------------------

class PCIeSoC(SoCCore):
    def __init__(self, sys_clk_freq = int(100e6), **kwargs):
        sys_clk_freq = int(100e6)

        platform = Platform()
        # SoCCore ----------------------------------------------------------------------------------
        SoCCore.__init__(self, platform, sys_clk_freq,
            ident          = "LiteX SoC on TE0712",
            **kwargs)

        # CRG --------------------------------------------------------------------------------------
        self.submodules.crg = CRG(platform, sys_clk_freq)

        # PCIe -------------------------------------------------------------------------------------
        # PHY
        self.submodules.pcie_phy = S7PCIEPHY(platform, platform.request("pcie_x1"),
            data_width = 64,
            bar0_size  = 0x20000)
        self.add_pcie(phy=self.pcie_phy, ndmas=1)
        # FIXME: Apply it to all targets (integrate it in LitePCIe?).
        platform.add_period_constraint(self.crg.cd_sys.clk, 1e9/sys_clk_freq)
        platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks userclk1] -asynchronous", sys_clk=self.crg.cd_sys.clk)
        platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks clk_125mhz] -asynchronous", sys_clk=self.crg.cd_sys.clk)
        platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks {sys_clk}] -group [get_clocks clk_250mhz] -asynchronous", sys_clk=self.crg.cd_sys.clk)
        platform.toolchain.pre_placement_commands.add("set_clock_groups -group [get_clocks clk_125mhz] -group [get_clocks clk_250mhz] -asynchronous")

# Create simple LED module (fpga description)
class Blink(Module):
    def __init__(self, blink_freq, sys_clk_freq, led):
        counter = Signal(32)
        # synchronous assignments
        self.sync += [
            counter.eq(counter + 1),
            If(counter == int((sys_clk_freq/blink_freq)/2 - 1),
                counter.eq(0),
                led.eq(~led)
            )
        ]
        # combinatorial assignements
        self.comb += []

class LEDModule(Module):
    def __init__(self, platform):
        pass
        # submodules
        self.submodules += Blink(1, 100e6, platform.request("sys_led"))
        self.submodules += Blink(2, 100e6, platform.request("led2"))

# Build --------------------------------------------------------------------------------------------

def main():
    parser = argparse.ArgumentParser(description="LiteX SoC on TE0712")
    parser.add_argument("--build",  action="store_true", help="Build bitstream")
    parser.add_argument("--driver", action="store_true", help="Generate LitePCIe driver")
    parser.add_argument("--load",   action="store_true", help="Load bitstream")
    parser.add_argument("--flash",  action="store_true", help="Flash bitstream")
    parser.add_argument("--sys-clk-freq",    default=100e6,       help="System clock frequency.")

    builder_args(parser)
    soc_core_args(parser)
    args = parser.parse_args()

    soc      = PCIeSoC(
        sys_clk_freq = int(float(args.sys_clk_freq)), 
        **soc_core_argdict(args))
    builder  = Builder(soc, **builder_argdict(args))
    builder.build(run=args.build)

    if args.driver:
        generate_litepcie_software(soc, os.path.join(builder.output_dir, "driver"))

    if args.load:
        prog = soc.platform.create_programmer()
        prog.load_bitstream(os.path.join(builder.gateware_dir, soc.build_name + ".bit"))

    if args.flash:
        prog = soc.platform.create_programmer()
        prog.load_bitstream(os.path.join(builder.gateware_dir, soc.build_name + ".bin"))

    # platform = Platform()
    # module = LEDModule(platform)
    # platform.build(module)

if __name__ == "__main__":
    main()

I can build the module using ./example_test.py --uart-name=crossover --build --driver --cpu-type None and it builds and generates the bitstream without any major problem. However, when I program the FPGA after the CPU reset, the PCIe Phy never shows up during boot. Below is the stdout during boot. The lspci returns nothing.

[    3.599568] imx6q-pcie 33800000.pcie: 33800000.pcie supply epdev_on not found, using dummy regulator
[    3.609235] OF: PCI: host bridge /pcie@0x33800000 ranges:
[    3.614659] OF: PCI:   No bus range found for /pcie@0x33800000, using [bus 00-ff]
[    3.622155] OF: PCI:    IO 0x1ff80000..0x1ff8ffff -> 0x00000000
[    3.628086] OF: PCI:   MEM 0x18000000..0x1fefffff -> 0x18000000
[    3.634493] imx6q-pcie 33800000.pcie: Initialize PHY with EXT REfCLK!.
[    3.641335] imx6q-pcie 33800000.pcie: PHY Initialization End!.
[    3.647502] imx6q-pcie 33800000.pcie: pcie phy pll is locked.
[    3.898406] imx6q-pcie 33800000.pcie: phy link never came up
[    3.904082] imx6q-pcie 33800000.pcie: failed to initialize host
[    3.910008] imx6q-pcie 33800000.pcie: unable to add pcie port.
[    3.916162] imx6q-pcie: probe of 33800000.pcie failed with error -110
[    3.924816] input: gpio-keys as /devices/platform/gpio-keys/input/input1
[    3.932480] rtc-m41t80 3-0068: Oscillator failure, data is invalid.
[    3.938762] rtc-m41t80 3-0068: hctosys: unable to read the hardware clock

I know my PCIe lines are right as I use the same pin configurations in a separate Vivado project and I can successfully connect to the Linux CPU through PCIe. Below is the same output when PCIe is recognized correctly.

[    3.600366] imx6q-pcie 33800000.pcie: 33800000.pcie supply epdev_on not found, using dummy regulator
[    3.610221] OF: PCI: host bridge /pcie@0x33800000 ranges:
[    3.615671] OF: PCI:   No bus range found for /pcie@0x33800000, using [bus 00-ff]
[    3.623186] OF: PCI:    IO 0x1ff80000..0x1ff8ffff -> 0x00000000
[    3.629139] OF: PCI:   MEM 0x18000000..0x1fefffff -> 0x18000000
[    3.635846] imx6q-pcie 33800000.pcie: Initialize PHY with EXT REfCLK!.
[    3.642702] imx6q-pcie 33800000.pcie: PHY Initialization End!.
[    3.648878] imx6q-pcie 33800000.pcie: pcie phy pll is locked.
[    3.709523] imx6q-pcie 33800000.pcie: Link up, Gen2
[    3.715866] imx6q-pcie 33800000.pcie: PCI host bridge to bus 0000:00
[    3.722262] pci_bus 0000:00: root bus resource [bus 00-ff]
[    3.727778] pci_bus 0000:00: root bus resource [io  0x0000-0xffff]
[    3.733983] pci_bus 0000:00: root bus resource [mem 0x18000000-0x1fefffff]
[    3.741536] pci 0000:01:00.0: enabling Extended Tags
[    3.758650] pci 0000:00:00.0: BAR 0: assigned [mem 0x18000000-0x180fffff 64bit]
[    3.765979] pci 0000:00:00.0: BAR 14: assigned [mem 0x18100000-0x181fffff]
[    3.772866] pci 0000:00:00.0: BAR 6: assigned [mem 0x18200000-0x1820ffff pref]
[    3.780099] pci 0000:01:00.0: BAR 0: assigned [mem 0x18100000-0x1810ffff]
[    3.786911] pci 0000:00:00.0: PCI bridge to [bus 01-ff]
[    3.792146] pci 0000:00:00.0:   bridge window [mem 0x18100000-0x181fffff]
[    3.799363] pcieport 0000:00:00.0: Signaling PME with IRQ 226
[    3.805205] pcieport 0000:00:00.0: AER enabled with IRQ 226
[    3.811129] serial 0000:01:00.0: enabling device (0000 -> 0002)

I am using NXP i.MX8 processor on the Linux side. What am I missing here? I am completely new to LitePCIe.

enjoy-digital commented 2 years ago

Hi @arindam-bose,

the code looks fine. The issue could be related to some specificities of the hardware/i.MX8. I would recommend looking a few things first:

Put a LiteX bridge in place to simplify the debug: https://github.com/enjoy-digital/litex/wiki/Use-Host-Bridge-to-control-debug-a-SoC
Observe the link status registers over the LiteX bridge: https://github.com/enjoy-digital/litepcie/blob/master/litepcie/phy/s7pciephy.py#L29-L45, doing a litex_cli --regs --filter=link_status should do a dump of this registers.
Try to comment the reset line in your code: #Subsignal("rst_n", Pins("M17"), IOStandard("LVCMOS33"), Misc("PULLUP=TRUE")),

arindam-bose commented 2 years ago

Thanks for the quick reply. I put an UART bridge as,

("serial_debug", 0,
        Subsignal("tx", Pins("U18")),
        Subsignal("rx", Pins("P16")),
        IOStandard("LVCMOS33")
    ),

and instantiate the UART in the SoC as self.add_uartbone(name="serial_debug", baudrate=115200) I generate the bitstream and program the FPGA. Then I start the litex_server using

$ litex_server --uart --uart-port=/dev/ttyUSB2
>> [CommUART] port: /dev/ttyUSB2 / baudrate: 115200 / tcp port: 1234

I get this when I dump the register as

$ litex_cli --regs
>> Traceback (most recent call last):
  File "/usr/local/bin/litex_cli", line 11, in <module>
    load_entry_point('litex', 'console_scripts', 'litex_cli')()
  File "/home/abose/tools/LiteX/litex/litex/tools/litex_client.py", line 166, in main
    dump_registers(port=port, filter=args.filter)
  File "/home/abose/tools/LiteX/litex/litex/tools/litex_client.py", line 121, in dump_registers
    if hasattr(bus.bases, "pcie_phy"):
AttributeError: 'RemoteClient' object has no attribute 'bases'

Does that mean I didn't instantiate the UART correctly?

arindam-bose commented 2 years ago

Also, I tried to remove the reset pin of PCIe: #Subsignal("rst_n", Pins("M17"), IOStandard("LVCMOS33"), Misc("PULLUP=TRUE")), but still the same issue persists , i.e., The PCIe was not enumerated.

enjoy-digital commented 2 years ago

@arindam-bose: Sorry I forgot to specify that you need to generate the SoC with --csr-csv=csr.csv and then run litex_cli in the repository with the csr.csv.

arindam-bose commented 2 years ago

Thanks @enjoy-digital, that was a great help. I had to install the latest OpenOCD to make LiteX-server working for some reason. Anyways, here is the hexdump of all the registers that I use in the csr.csv.

0x00000000 : 0x00000000 ctrl_reset
0x00000004 : 0x12345678 ctrl_scratch
0x00000008 : 0x00000000 ctrl_bus_errors
0x00000800 : 0x00000000 flash_spi_control
0x00000804 : 0x00000001 flash_spi_status
0x00000808 : 0x00000000 flash_spi_mosi
0x00000810 : 0x00000000 flash_spi_miso
0x00000818 : 0x00000001 flash_spi_cs
0x0000081c : 0x00000000 flash_spi_loopback
0x00001000 : 0x00000000 flash_cs_n_out
0x00001800 : 0x00000000 icap_addr
0x00001804 : 0x00000000 icap_data
0x00001808 : 0x00000000 icap_write
0x0000180c : 0x00000000 icap_done
0x00001810 : 0x00000000 icap_read
0x00002800 : 0x00000000 pcie_dma0_writer_enable
0x00002804 : 0x00000000 pcie_dma0_writer_table_value
0x0000280c : 0x00000000 pcie_dma0_writer_table_we
0x00002810 : 0x00000000 pcie_dma0_writer_table_loop_prog_n
0x00002814 : 0x00000000 pcie_dma0_writer_table_loop_status
0x00002818 : 0x00000000 pcie_dma0_writer_table_level
0x0000281c : 0x00000000 pcie_dma0_writer_table_reset
0x00002820 : 0x00000000 pcie_dma0_reader_enable
0x00002824 : 0x00000000 pcie_dma0_reader_table_value
0x0000282c : 0x00000000 pcie_dma0_reader_table_we
0x00002830 : 0x00000000 pcie_dma0_reader_table_loop_prog_n
0x00002834 : 0x00000000 pcie_dma0_reader_table_loop_status
0x00002838 : 0x00000000 pcie_dma0_reader_table_level
0x0000283c : 0x00000000 pcie_dma0_reader_table_reset
0x00002840 : 0x00000000 pcie_dma0_loopback_enable
0x00002844 : 0x00000400 pcie_dma0_buffering_reader_fifo_control
0x00002848 : 0x00000000 pcie_dma0_buffering_reader_fifo_status
0x0000284c : 0x00000400 pcie_dma0_buffering_writer_fifo_control
0x00002850 : 0x00000000 pcie_dma0_buffering_writer_fifo_status
0x00003000 : 0x00000000 pcie_dma1_writer_enable
0x00003004 : 0x00000000 pcie_dma1_writer_table_value
0x0000300c : 0x00000000 pcie_dma1_writer_table_we
0x00003010 : 0x00000000 pcie_dma1_writer_table_loop_prog_n
0x00003014 : 0x00000000 pcie_dma1_writer_table_loop_status
0x00003018 : 0x00000000 pcie_dma1_writer_table_level
0x0000301c : 0x00000000 pcie_dma1_writer_table_reset
0x00003020 : 0x00000000 pcie_dma1_reader_enable
0x00003024 : 0x00000000 pcie_dma1_reader_table_value
0x0000302c : 0x00000000 pcie_dma1_reader_table_we
0x00003030 : 0x00000000 pcie_dma1_reader_table_loop_prog_n
0x00003034 : 0x00000000 pcie_dma1_reader_table_loop_status
0x00003038 : 0x00000000 pcie_dma1_reader_table_level
0x0000303c : 0x00000000 pcie_dma1_reader_table_reset
0x00003040 : 0x00000000 pcie_dma1_loopback_enable
0x00003044 : 0x00000400 pcie_dma1_buffering_reader_fifo_control
0x00003048 : 0x00000000 pcie_dma1_buffering_reader_fifo_status
0x0000304c : 0x00000400 pcie_dma1_buffering_writer_fifo_control
0x00003050 : 0x00000000 pcie_dma1_buffering_writer_fifo_status
0x00003800 : 0x00000000 pcie_msi_enable
0x00003804 : 0x00000000 pcie_msi_clear
0x00003808 : 0x00000000 pcie_msi_vector
0x00004000 : 0x00000000 pcie_phy_link_status
0x00004004 : 0x00000000 pcie_phy_msi_enable
0x00004008 : 0x00000000 pcie_phy_msix_enable
0x0000400c : 0x00000000 pcie_phy_bus_master_enable
0x00004010 : 0x00000200 pcie_phy_max_request_size
0x00004014 : 0x00000080 pcie_phy_max_payload_size
0x00004800 : 0x00000000 timer0_load
0x00004804 : 0x00000000 timer0_reload
0x00004808 : 0x00000000 timer0_en
0x0000480c : 0x00000000 timer0_update_value
0x00004810 : 0x00000000 timer0_value
0x00004814 : 0x00000001 timer0_ev_status
0x00004818 : 0x00000001 timer0_ev_pending
0x0000481c : 0x00000000 timer0_ev_enable
0x00005000 : 0x00000000 uart_rxtx
0x00005004 : 0x00000000 uart_txfull
0x00005008 : 0x00000001 uart_rxempty
0x0000500c : 0x00000001 uart_ev_status
0x00005010 : 0x00000001 uart_ev_pending
0x00005014 : 0x00000000 uart_ev_enable
0x00005018 : 0x00000001 uart_txempty
0x0000501c : 0x00000000 uart_rxfull
0x00005020 : 0x00000000 uart_xover_rxtx
0x00005024 : 0x00000000 uart_xover_txfull
0x00005028 : 0x00000001 uart_xover_rxempty
0x0000502c : 0x00000001 uart_xover_ev_status
0x00005030 : 0x00000001 uart_xover_ev_pending
0x00005034 : 0x00000000 uart_xover_ev_enable
0x00005038 : 0x00000001 uart_xover_txempty
0x0000503c : 0x00000000 uart_xover_rxfull

And especially, with

$ litex_cli --regs --filter=link_status
>> 0x00004000 : 0x00000000 pcie_phy_link_status

In particular, according to this, does this mean we have 2.5 Gb/s 1-Lane link, which is down and the LTSSM State is 0x00? Furthermore, the value of the same register before booting up the Linux is 0x0000000C, if that is helpful. I will remove the PCIe reset line and try it again.

enjoy-digital commented 2 years ago

@arindam-bose: It could be useful to do a test with https://github.com/enjoy-digital/litepcie/commit/b8ff82a6c3c614237db14519647c0d191f492961.

arindam-bose commented 2 years ago

Thanks, I tried that. But unfortunately, that wasn't the cause of the problem. The problem lies in the default placement of the Tx/Rx lane pins of the PCIe IP. In my design, I am using PCIe_Gen2_x1 mode and that is why I used

# pcie
    ("pcie_x1", 0,
        Subsignal("rst_n", Pins("M17"), IOStandard("LVCMOS33"), Misc("PULLUP=TRUE")),
        Subsignal("clk_p", Pins("F10")),
        Subsignal("clk_n", Pins("E10")),
        Subsignal("rx_p",  Pins("B8")),
        Subsignal("rx_n",  Pins("A8")),
        Subsignal("tx_p",  Pins("B4")),
        Subsignal("tx_n",  Pins("A4")),
    ),

However, when the PCIe IP is generated, it selects GTPE2_CHANNEL_X0Y7 as the default x1 PCIe channel, and thus later during implementation, the IO maps to other pins relevant to this GTPE2_CHANNEL_X0Y7. This setting can be found in project.gen/sources_1/ip/pcie_s7/source/pcie_s7-PCIE_X0Y0.xdc file line 94 as, set_property LOC GTPE2_CHANNEL_X0Y7 [get_cells {inst/gt_top_i/pipe_wrapper_i/pipe_lane[0].gt_wrapper_i/gtp_channel.gtpe2_channel_i}]

I found this innocent error while checking through the generated io report. Once I changed it to GTPE2_CHANNEL_X0Y4, it worked. Basically, everything was fine, except for the default PCIe lane placement which ignores the provided IO pin mapping.

Can you tell me, how to avoid this default channel selection.

enjoy-digital commented 2 years ago

@arindam-bose: Good that you figured out the issue, I also had similar issue with constraining the PCIe IO in the past. Can you try to change it here and see if the design builds and behaves correctly?: https://github.com/enjoy-digital/litepcie/blob/master/litepcie/phy/s7pciephy.py#L405

If so, we could expose this parameter to allow configuration by the user.

arindam-bose commented 2 years ago

@enjoy-digital: Unfortunately you can't change that. I mean if you do the design won't synthesize.

julianstj1 commented 2 years ago

How did you force it to override to X0Y4? I'm running into the same issue building for my design.

EDIT: Nevermind, I figured it out. Added these lines to s7pciephy.py

    # Reset LOC constraints on GTPE2_COMMON and BRAM36 from .xci (we only want to keep Timing constraints).
    if self.pcie_gt_device == "GTP":
        platform.toolchain.pre_placement_commands.append("reset_property LOC [get_cells -hierarchical -filter {{NAME=~pcie_support/*gtp_common.gtpe2_common_i}}]")
        if self.nlanes == 1:
            platform.toolchain.pre_placement_commands.append("reset_property LOC [get_cells -hierarchical -filter {{NAME=~pcie_support/*gtp_channel.gtpe2_channel_i}}]")
            platform.toolchain.pre_placement_commands.append("set_property LOC GTPE2_CHANNEL_X0Y4 [get_cells -hierarchical -filter {{NAME=~pcie_support/*gtp_channel.gtpe2_channel_i}}]")

enjoy-digital / litepcie

PCIe bus never comes alive #87