[UHD Python API] Unable to synchronize two x310 in PPS

I am trying to acquire samples with the following equipment:

Two x310 (Dual 10GbE each), each x310 has internally two TwinRX and a meimberg card for reference signal generation and PPS. The oscillators of the first twinRX are exported via hardware with splitters to the rest of the twinRX (as well as to itself back so that they all have the same delay).

I am using UHD version 4.6.0.0.0 and the Python API.

Here is the code I have implemented:

import numpy as np
import uhd
import gc
import subprocess
import datetime
import os
from multiprocessing import Process
import time

# Hardcoded parameters
output_file = "outputfile_name"
freq = 1e9
rate = np.float128(100e6/3)
gain = 60
duration = 1
channels = [0, 1, 2, 3, 4, 5]
addr0 = "192.168.60.2"
second_addr0 = "192.168.50.2"
addr1 = "192.168.40.2"
second_addr1 = "192.168.30.2"

def create_directory(base_path, filename, current_time):
    """Create a directory with the base path and filename"""
    directory = os.path.join(base_path, f"{filename}__{current_time}")
    if not os.path.exists(directory):
        os.makedirs(directory)
    return directory

def save_samples_to_file(result, channels, directory, current_time):
    """Save samples to files in the specified directory"""
    for i, channel_id in enumerate(channels):
        output_filename = os.path.join(directory, f"{output_file}__{current_time}__CH{channel_id}.iq")
        print(f"Saving samples for channel {channel_id} to {output_filename}")
        result[i].tofile(output_filename)

def acquire_and_save_samples(addr0, second_addr0, addr1, second_addr1, freq, rate, gain, duration, channels):
    """Acquire samples from USRP and save to file"""
    try:
        usrp = uhd.usrp.MultiUSRP(f"addr0={addr0}, second_addr0={second_addr0},addr1={addr1}, second_addr1={second_addr1}, num_recv_frames=1000, recv_frame_size=9216, recv_buff_size=1073741823")
        # Clock configuration
        usrp.set_clock_source("external")

        # Subdevice configuration
        usrp.set_rx_subdev_spec(uhd.usrp.SubdevSpec("A:0 A:1 B:0 B:1"), 0)
        usrp.set_rx_subdev_spec(uhd.usrp.SubdevSpec("A:0 A:1"), 1)

        # Parameter configuration
        for i in channels:
            usrp.set_rx_rate(rate, i)
            usrp.set_rx_bandwidth(rate, i)
            usrp.set_rx_freq(uhd.libpyuhd.types.tune_request(freq), i)
            usrp.set_rx_gain(gain, i)
        num_samps = int(np.ceil(duration * rate))

        # LO configuration
        usrp.set_rx_lo_export_enabled(True, "all", 0)
        usrp.set_rx_lo_source("internal", "all", 0)
        usrp.set_rx_lo_source("companion", "all", 1)
        num_rx_channels = usrp.get_rx_num_channels()
        for i in range(2, num_rx_channels):
            usrp.set_rx_lo_source("external", "all", i)
        for i in range(num_rx_channels):
            print(f"Channel {i}:")
            print(f"Enable LO: {usrp.get_rx_lo_export_enabled('all', i)}")
            print(f"LO1 freq: {usrp.get_rx_lo_freq('LO1', i) / 1e9} GHz")
            print(f"LO2 freq: {usrp.get_rx_lo_freq('LO2', i) / 1e9} GHz")
            print(f"LO source: {usrp.get_rx_lo_source('all', i)}")
            print()

        # Time configuration
        usrp.set_time_source("external")

        # Buffer setup
        st_args = uhd.usrp.StreamArgs("sc16", "sc16")
        st_args.channels = channels
        streamer = usrp.get_rx_stream(st_args)
        metadata = uhd.types.RXMetadata()

        # Receive samples
        result = np.empty((len(channels), num_samps), dtype=np.int32)
        recv_buffer = np.zeros((len(channels), streamer.get_max_num_samps()), dtype=np.int32)
        recv_samps = 0

        # Start stream
        current_time = datetime.datetime.now().strftime("%d_%m_%Y__%H_%M_%S")
        stream_cmd = uhd.types.StreamCMD(uhd.types.StreamMode.start_cont)
        stream_cmd.stream_now = (len(channels) == 1)
        if not stream_cmd.stream_now:
            stream_cmd.time_spec = uhd.types.TimeSpec(usrp.get_time_now().get_real_secs() + 0.05)
        streamer.issue_stream_cmd(stream_cmd)

        while recv_samps < num_samps:
            samps = streamer.recv(recv_buffer, metadata)
            if metadata.error_code != uhd.types.RXMetadataErrorCode.none:
                print(metadata.strerror())
            if samps:
                real_samps = min(num_samps - recv_samps, samps)
                result[:, recv_samps:recv_samps + real_samps] = recv_buffer[:, 0:real_samps]
                recv_samps += real_samps

        # Stop stream
        stream_cmd = uhd.types.StreamCMD(uhd.types.StreamMode.stop_cont)
        streamer.issue_stream_cmd(stream_cmd)
        while streamer.recv(recv_buffer, metadata):
            pass

        #print(f"Bandwidth: {usrp.get_rx_bandwidth(0)/1e6:.0f} MHz")
        print(f"Bandwidth: {usrp.get_rx_bandwidth(0)} MHz")
        print(f"Format of samples: {type(result[0, 1])}")
        print(f"Shape of samples: {result.shape}")
        print(f"Number of channels: {result.shape[0]}")
        print(f"Number of samples per channel: {result.shape[1]}")
        print(f"Size per .iq file: {result.dtype.itemsize * num_samps / 1e6:.0f} MB")

        # Here is a code block to save the samples in .iq files (omitted to save space)

    except Exception as e:
        print(f"Error occurred: {e}")

def main():
    """Main function to handle sample acquisition"""
    acquire_and_save_samples(addr0, second_addr0, addr1, second_addr1, freq, rate, gain, duration, channels)

if __name__ == "__main__":
    main()

The problem is the following: All channels of the same x310 are synchronized with each other, but when I try to check the synchronization of a channel of one card with another channel of the other card, they are not synchronized. Regarding this I have read the following: “set the time at a PPS edge to the same time on both devices” and “only use timed commands to start and stop the stream”, but I am not sure how to implement it.

I have also heard that this problem is happening to more people, has it happened to you?

Could it be related to the UHD version or the Python API?

EttusResearch / uhd

[UHD Python API] Unable to synchronize two x310 in PPS #791