Low goodput and strange behavior at high bottleneck link bandwidths (100Mbps and 1000Mbps)

[MCxiaofang]

I wanted to use the program to support testing of 100Mbps and 1000Mbps (40ms rtt)bottleneck link bandwidths, but the results performed abnormally at such high bandwidths, as summarized below。

1. Goodput is only 1500Kbps (whether 100Mbps or 1000Mbps bottleneck link bandwidth)

2. The sendrate, however, increases as the bandwidth of the bottleneck link increases (57622Kbps sendrate when the goodput is only 1500kbps)

   

3. The final statistic has a very low packet loss rate

4. NS3 simulation runs extremely fast (compared to a program that also runs normally with TCP at 100Mbps bottleneck link bandwidth) (even faster than a quic test program running at 10Mbps bottleneck link bandwidth) (thus proving that it's not the stats program that's at fault)

5. sendrate stops counting after 5.3s, according to the trace program, we found that it is because if (m_rate ! = bps) caused sendrate.txt to stop updating because the bandwidth calculated by the congestion control algorithm has remained constant since 5.3s. (The program calculates the send rate by dividing the congestion window size by srtt, and the cout reveals that the values of srtt and cwnd do not change after 5.3s.)

Notice that the results are partially normal after making the following changes on 100Mbps（1000Mbps always strange）, which can be used to further speculate on the root cause of the anomalous performance。

1. The copa algorithm works fine (including things like goodput and NS3 simulation speed), but cubic and bbrv2 don't!

   ./waf --run "scratch/dumbbell-quic --cc1=bbrv2 --cc2=bbrv2 --lo=0 --nLeaf=4 --gap=2 --it=2"

   ./waf --run "scratch/dumbbell-quic --cc1=copa --cc2=copa --lo=0 --nLeaf=4 --gap=2 --it=2"

2. The cubic algorithms also work fine when a very high packet loss rate is set (1%)(bbrv2 still not) But it's not exactly normal either, although it's not a thousand 1.5Mbps anymore, it's only 4 to 6Mbps at 100Mbps

   ./waf --run "scratch/dumbbell-quic --cc1=cubic --cc2=cubic --lo=1000 --nLeaf=4 --gap=2 --it=2"

Here is the source code for my test file，Run the test file with the following command ./waf --run "scratch/dumbbell-quic --cc1=bbrv2 --cc2=cubic --lo=0 --nLeaf=4 --gap=2 --it=2" nLeaf: Specify the number of single-ended user nodes for dumbbell topologies cc1: Congestion control algorithm used by half of the nodes in a prescribed dumbbell topology cc2: Specify the congestion control algorithm to be used by the nodes in the other half of the dumbbell topology gap: Interval between application launches(first app start at 5.0s) lo: Packet loss rate, 10 equals 0.01%, 100 equals 01%

#include <iostream>
#include <stdlib.h>
#include <memory>
#include <string>
#include <unistd.h>
#include "ns3/core-module.h"
#include "ns3/applications-module.h"
#include "ns3/internet-module.h"
#include "ns3/network-module.h"
#include "ns3/point-to-point-module.h"
#include "ns3/ipv4-global-routing-helper.h"
#include "ns3/traffic-control-module.h"
#include "ns3/quic-module.h"
#include "ns3/nstime.h"
#include "ns3/point-to-point-layout-module.h"

using namespace ns3;
using namespace quic;
using namespace std;
NS_LOG_COMPONENT_DEFINE("dumbbell-quic");
const uint32_t DEFAULT_PACKET_SIZE = 1500;

static const float appStart = 5;
static const float appStop =20.0;

// 每秒打印一次当前系统时间, 用于监控仿真进度
void monitor(double begin_time)
{
    NS_LOG_UNCOND((Simulator::Now ().GetSeconds () - begin_time) << "s");
    Simulator::Schedule(Seconds(1), &monitor, begin_time);
}

class TriggerRandomLoss
{
public:
    TriggerRandomLoss() {}
    ~TriggerRandomLoss()
    {
        if (m_timer.IsRunning())
        {
            m_timer.Cancel();
        }
    }
    void RegisterDevice(Ptr<NetDevice> dev)
    {
        m_dev = dev;
    }
    void Start()
    {
        Time next = Seconds(2);
        m_timer = Simulator::Schedule(next, &TriggerRandomLoss::ConfigureRandomLoss, this);
    }
    void ConfigureRandomLoss()
    {
        if (m_timer.IsExpired())
        {
            std::string errorModelType = "ns3::RateErrorModel";
            ObjectFactory factory;
            factory.SetTypeId(errorModelType);
            Ptr<ErrorModel> em = factory.Create<ErrorModel>();
            m_dev->SetAttribute("ReceiveErrorModel", PointerValue(em));
            m_timer.Cancel();
        }
    }

private:
    Ptr<NetDevice> m_dev;
    EventId m_timer;
};

void getInstance(std::string &instance,
                 uint64_t &bottle_rate, uint64_t &access_rate,
                 uint64_t &bottle_delay, uint64_t &access_delay)
{
    if (instance.compare("1") == 0)
    {
        bottle_rate = 10 * 1e6;
        bottle_delay = 40;
        access_rate = 1000 * 1e6;
        access_delay = 1;
    }
    else if (instance.compare("2") == 0)
    {
        bottle_rate = 100 * 1e6;
        bottle_delay = 40;
        access_rate = 1000 * 1e6;
        access_delay = 1;
    }
    else if (instance.compare("3") == 0)
    {
        bottle_rate = 1000 * 1e6;
        bottle_delay = 40;
        access_rate = 1000 * 1e6;
        access_delay = 1;
    }
}

void test_app_on_dumbbell(std::string &instance, uint8_t client_log_flag, uint8_t server_log_flag,
                          quic::BackendType type, TriggerRandomLoss *trigger_loss,
                          const std::string &cc1, const std::string &cc2, uint32_t nLeaf, uint32_t gap)
{
    uint64_t bottle_rate, bottle_delay;
    uint64_t access_rate, access_delay;

    getInstance(instance, bottle_rate, access_rate, bottle_delay, access_delay);
    //auto bufSize = std::max<uint32_t>(DEFAULT_PACKET_SIZE, bottle_rate / 8000 *  bottle_delay);
    //int BDP = bufSize / DEFAULT_PACKET_SIZE;

    // 输出各项配置
    NS_LOG_UNCOND("bottle_rate: " << bottle_rate / 1e6 << "Mbps");
    NS_LOG_UNCOND("access_rate: " << access_rate / 1e6 << "Mbps");
    NS_LOG_UNCOND("RTT: " << bottle_delay << "ms");
    // NS_LOG_UNCOND("BDP: " << BDP);
    NS_LOG_UNCOND("cc1: " << cc1);
    NS_LOG_UNCOND("cc1: " << cc2);

    PointToPointHelper bottleNeckLink;
    bottleNeckLink.SetDeviceAttribute("DataRate", DataRateValue(DataRate(bottle_rate)));
    bottleNeckLink.SetChannelAttribute("Delay", TimeValue(MilliSeconds(bottle_delay / 2)));
    bottleNeckLink.SetQueue("ns3::DropTailQueue", "MaxSize", StringValue("3p"));

    PointToPointHelper pointToPointLeaf;
    pointToPointLeaf.SetDeviceAttribute("DataRate", DataRateValue(DataRate(access_rate)));
    pointToPointLeaf.SetChannelAttribute("Delay", TimeValue(MilliSeconds(access_delay)));
    pointToPointLeaf.SetQueue("ns3::DropTailQueue", "MaxSize", StringValue("3p"));

    NodeContainer routers, leftLeaf, rightLeaf;
    NetDeviceContainer routerDevices, 
            leftLeafDevices, rightLeafDevices, 
            leftRouterDevices, rightRouterDevices;

    routers.Create(2);
    leftLeaf.Create(nLeaf);
    rightLeaf.Create(nLeaf);

    // Install Stack
    InternetStackHelper stack;
    stack.Install(routers);
    stack.Install(leftLeaf);
    stack.Install(rightLeaf);

    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        NetDeviceContainer c = pointToPointLeaf.Install(routers.Get(0), leftLeaf.Get(i));
        leftRouterDevices.Add(c.Get(0));
        leftLeafDevices.Add(c.Get(1));
    }
    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        NetDeviceContainer c = pointToPointLeaf.Install(routers.Get(1), rightLeaf.Get(i));
        rightRouterDevices.Add(c.Get(0));
        rightLeafDevices.Add(c.Get(1));
    }

    routerDevices = bottleNeckLink.Install(routers);

    TrafficControlHelper tchFq;
    tchFq.SetRootQueueDisc("ns3::FqCoDelQueueDisc");
    tchFq.SetQueueLimits("ns3::DynamicQueueLimits", "HoldTime", StringValue("1ms"));
    tchFq.Install (leftRouterDevices);
    tchFq.Install (leftLeafDevices);
    tchFq.Install (rightRouterDevices);
    tchFq.Install (rightLeafDevices);
    tchFq.Install (routerDevices);

    //TrafficControlHelper pfifoHelper;
    //uint16_t handle = pfifoHelper.SetRootQueueDisc("ns3::FifoQueueDisc", "MaxSize", StringValue(std::to_string(400) + "p"));
    //pfifoHelper.AddInternalQueues(handle, 1, "ns3::DropTailQueue", "MaxSize", StringValue(std::to_string(400) + "p"));
    //pfifoHelper.Install(routerDevices);

    if(trigger_loss){
            trigger_loss->RegisterDevice(routerDevices.Get(1));
    }

    Ipv4AddressHelper leftIp("10.1.1.0", "255.255.255.0");
    Ipv4AddressHelper rightIp("10.2.1.0", "255.255.255.0");
    Ipv4AddressHelper routerIp("10.3.1.0", "255.255.255.0");

    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        NetDeviceContainer ndc;
        ndc.Add(leftRouterDevices.Get(i));
        ndc.Add(leftLeafDevices.Get(i));
        leftIp.Assign(ndc);
        leftIp.NewNetwork();
    }
    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        NetDeviceContainer ndc;
        ndc.Add(rightRouterDevices.Get(i));
        ndc.Add(rightLeafDevices.Get(i));
        rightIp.Assign(ndc);
        rightIp.NewNetwork();
    }
    routerIp.Assign(routerDevices);

    Ipv4GlobalRoutingHelper::PopulateRoutingTables ();

    std::vector<Ns3QuicClientTrace *> client_traces;
    Ns3QuicServerTraceDispatcher *server_trace = nullptr;
    if (server_log_flag & E_QS_ALL)
    {
        server_trace = new Ns3QuicServerTraceDispatcher();
        server_trace->LogEnable(instance, server_log_flag); // instance值仅用来设置文件名前缀
    }

    uint16_t server_port = 1234;
    std::vector<InetSocketAddress> server_addrs;

    // install server(recv msg) on right leaf
    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        Ptr<Node> host = rightLeaf.Get(i);
        Ptr<QuicServerApp> server_app = CreateObject<QuicServerApp>(type);
        host->AddApplication(server_app);

        server_app->Bind(server_port);
        server_addrs.push_back(server_app->GetLocalAddress());        // get server address
        server_app->SetStartTime(Seconds(appStart + i * gap));
        server_app->SetStopTime(Seconds(appStop));
        if (server_trace)
        {
            server_app->set_trace(server_trace);
        }
    }

    // install client on left leaf
    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        Ptr<Node> host = leftLeaf.Get(i);
        Ptr<QuicClientApp> client_app;
        if(i < nLeaf / 2){
            client_app = CreateObject<QuicClientApp>(type, cc1.c_str());
        }
        else{
            client_app = CreateObject<QuicClientApp>(type, cc2.c_str());
        }
        host->AddApplication(client_app);

        client_app->Bind();
        InetSocketAddress client_addr = client_app->GetLocalAddress();
        client_app->set_peer(server_addrs[i].GetIpv4(), server_addrs[i].GetPort());
        client_app->SetStartTime(Seconds(appStart + i * gap));
        client_app->SetStopTime(Seconds(appStop));

        Ns3QuicAddressPair addr_pair(client_addr, server_addrs[i]);
        if (client_log_flag & E_QC_ALL)
        {
            std::string prefix = instance + "_" + Ns3QuicAddressPair2String(addr_pair);
            Ns3QuicClientTrace *trace = new Ns3QuicClientTrace();
            trace->LogEnable(prefix, client_log_flag);
            if (client_log_flag & E_QC_IN_FLIGHT)
            {
                client_app->SetInFlightTraceFun(
                    MakeCallback(&Ns3QuicClientTrace::OnInFlight, trace));
            }
            if (client_log_flag & E_QC_SEND_RATE)
            {
                client_app->SetRateTraceFuc(MakeCallback(&Ns3QuicClientTrace::OnSendRate, trace));
            }
            client_traces.push_back(trace);
        }
        if (server_log_flag & E_QS_ALL)
        {
            server_trace->AddMonitorAddressPair(addr_pair);
        }
    }

    Simulator::Schedule(Seconds(0.0), &monitor, Simulator::Now ().GetSeconds ());

    bottleNeckLink.EnablePcap("dumbbell", routerDevices.Get(1), true);

    int last_time = WallTimeMillis();
    Simulator::Stop(Seconds(appStop) + Seconds(10));
    Simulator::Run();
    Simulator::Destroy();
    for (auto it = client_traces.begin(); it != client_traces.end(); it++)
    {
        Ns3QuicClientTrace *ptr = (*it);
        delete ptr;
    }
    client_traces.clear();
    int delta = WallTimeMillis() - last_time;
    std::cout << "run time s: " << delta / 1000.0 << std::endl;
}

int main(int argc, char *argv[])
{
    std::string instance = std::string("1");
    std::string loss_str("0"); // config random loss
    std::string cc1("cubic");
    std::string cc2("cubic");
    std::string data_folder("no-one");
    uint32_t nLeaf = 4;
    uint32_t start_gap = 0;

    CommandLine cmd;
    cmd.AddValue("it", "instacne", instance);
    cmd.AddValue("folder", "folder name to collect data", data_folder);
    cmd.AddValue("lo", "loss", loss_str); // 10 means the dev will introduce 10/1000 % random loss
    cmd.AddValue("cc1", "congestion algorithm1", cc1);
    cmd.AddValue("cc2", "congestion algorithm2", cc2);
    cmd.AddValue("nLeaf", "number of leaf", nLeaf);
    cmd.AddValue("gap", "gap between application start", start_gap);
    cmd.Parse(argc, argv);

    Config::SetDefault ("ns3::UdpSocket::RcvBufSize",UintegerValue (100768000));

    int loss_integer = std::stoi(loss_str);
    double random_loss = loss_integer * 1.0 / 100000;
    std::unique_ptr<TriggerRandomLoss> triggerloss = nullptr;
    if (loss_integer > 0)
    {
        Config::SetDefault("ns3::RateErrorModel::ErrorRate", DoubleValue(random_loss));
        Config::SetDefault("ns3::RateErrorModel::ErrorUnit", StringValue("ERROR_UNIT_PACKET"));
        Config::SetDefault("ns3::BurstErrorModel::ErrorRate", DoubleValue(random_loss));
        Config::SetDefault("ns3::BurstErrorModel::BurstSize", StringValue("ns3::UniformRandomVariable[Min=1|Max=3]"));
        triggerloss.reset(new TriggerRandomLoss());         //reset是unique_ptr的函数，等同于赋值
        triggerloss->Start();
    }
    NS_LOG_UNCOND("random_loss: " << random_loss * 100 << "%");

    // 判断拥塞控制算法输入合法性
    if (0 == cc1.compare("reno") || 0 == cc1.compare("cubic") ||
        0 == cc1.compare("bbr") || 0 == cc1.compare("bbrv2") ||
        0 == cc1.compare("copa") || 0 == cc1.compare("vegas"))
    {
    }
    else
    {
        NS_ASSERT_MSG(0, "please input correct cc1");
    }
    if (0 == cc2.compare("reno") || 0 == cc2.compare("cubic") ||
        0 == cc2.compare("bbr") || 0 == cc2.compare("bbrv2") ||
        0 == cc2.compare("copa") || 0 == cc2.compare("vegas"))
    {
    }
    else
    {
        NS_ASSERT_MSG(0, "please input correct cc2");
    }

    // 检查QUICHE依赖
    const char *envKey = "QUICHE_SRC_DIR";
    char *envValue = getenv(envKey);
    if (envValue)
        NS_LOG_UNCOND("QUICHE_SRC_DIR:" << envValue);

    // 创建结果目录文件夹
    char buffer[128] = {0};
    if (getcwd(buffer, sizeof(buffer)) != buffer)
    {
        NS_LOG_ERROR("path error");
        return -1;
    }

    std::string ns3_path(buffer, ::strlen(buffer));
    if ('/' != ns3_path.back())
    {
        ns3_path.push_back('/');
    }
    std::string trace_folder = ns3_path + "traces/";
    NS_LOG_UNCOND("trace_folder: " << trace_folder);
    if (!MakePath(trace_folder))
    {
        std::cout << trace_folder << " is not right" << std::endl;
        return -1;
    }

    std::string log_path = trace_folder + data_folder + "/";
    set_quic_trace_folder(log_path);
    if (!MakePath(log_path))
    {
        std::cout << log_path << " is not right" << std::endl;
        return -1;
    }
    NS_LOG_UNCOND("log_path: " << log_path);

    set_quic_log_error_level(); // print quiche lib error
    set_quic_goodput_interval(MilliSeconds(200));
    quic::ContentInit('a');     // QUIC协议数据负载内容（全部填写a）

    std::string quic_cert_path = std::string(envValue) + "utils/data/quic-cert/";
    if (ns3::set_quic_cert_key(quic_cert_path))
    {
        quic::BackendType type = quic::BackendType::BANDWIDTH;

        QuicClientTraceType client_log_flag = E_QC_IN_FLIGHT | E_QC_SEND_RATE;
        QuicServerTraceType server_log_flag = E_QS_OWD | E_QS_LOST | E_QS_GOODPUT;

        RegisterExternalCongestionFactory();

        test_app_on_dumbbell(
            instance, // 链路属性
            client_log_flag, // Client负责接收数据，追踪inflight，发送速率
            server_log_flag, // Server负责接收数据，追踪单向时延，丢包率，接受速率
            type,
            triggerloss.get(),
            cc1,
            cc2,
            nLeaf,
            start_gap);
    }
    return 0;
}

To demonstrate that it is the other configurations in NS3 that do not affect the results, the following code is simply replacing the application in the above test file with the native application in NS3 that is tested using TCP。 It can be observed that the rest of the code is completely unchanged except for the application section。 Strange point: but again, previous test results proved that the TrafficControl policy still affects the results (the algorithm works fine at 100Mbps after switching from DynamicQueueLimits to a DropTailQueue with a low Maxsize)

#include <iostream>
#include <stdlib.h>
#include <memory>
#include <string>
#include <unistd.h>
#include "ns3/core-module.h"
#include "ns3/applications-module.h"
#include "ns3/internet-module.h"
#include "ns3/network-module.h"
#include "ns3/point-to-point-module.h"
#include "ns3/ipv4-global-routing-helper.h"
#include "ns3/traffic-control-module.h"
#include "ns3/quic-module.h"
#include "ns3/nstime.h"
#include "ns3/point-to-point-layout-module.h"

using namespace ns3;
using namespace quic;
using namespace std;
NS_LOG_COMPONENT_DEFINE("dumbbell");
const uint32_t DEFAULT_PACKET_SIZE = 1500;

static const float appStart = 5;
static const float appStop = 100.0;

// 每秒打印一次当前系统时间, 用于监控仿真进度
void monitor(double begin_time)
{
    NS_LOG_UNCOND((Simulator::Now ().GetSeconds () - begin_time) << "s");
    Simulator::Schedule(Seconds(1), &monitor, begin_time);
}

class TriggerRandomLoss
{
public:
    TriggerRandomLoss() {}
    ~TriggerRandomLoss()
    {
        if (m_timer.IsRunning())
        {
            m_timer.Cancel();
        }
    }
    void RegisterDevice(Ptr<NetDevice> dev)
    {
        m_dev = dev;
    }
    void Start()
    {
        Time next = Seconds(2);
        m_timer = Simulator::Schedule(next, &TriggerRandomLoss::ConfigureRandomLoss, this);
    }
    void ConfigureRandomLoss()
    {
        if (m_timer.IsExpired())
        {
            std::string errorModelType = "ns3::RateErrorModel";
            ObjectFactory factory;
            factory.SetTypeId(errorModelType);
            Ptr<ErrorModel> em = factory.Create<ErrorModel>();
            m_dev->SetAttribute("ReceiveErrorModel", PointerValue(em));
            m_timer.Cancel();
        }
    }

private:
    Ptr<NetDevice> m_dev;
    EventId m_timer;
};

void getInstance(std::string &instance,
                 uint64_t &bottle_rate, uint64_t &access_rate,
                 uint64_t &bottle_delay, uint64_t &access_delay)
{
    if (instance.compare("1") == 0)
    {
        bottle_rate = 10 * 1e6;
        bottle_delay = 40;
        access_rate = 1000 * 1e6;
        access_delay = 1;
    }
    else if (instance.compare("2") == 0)
    {
        bottle_rate = 100 * 1e6;
        bottle_delay = 40;
        access_rate = 1000 * 1e6;
        access_delay = 1;
    }
    else if (instance.compare("3") == 0)
    {
        bottle_rate = 1000 * 1e6;
        bottle_delay = 40;
        access_rate = 1000 * 1e6;
        access_delay = 1;
    }
}

void test_app_on_dumbbell(std::string &instance, uint8_t client_log_flag, uint8_t server_log_flag,
                          quic::BackendType type, TriggerRandomLoss *trigger_loss,
                          const std::string &cc1, const std::string &cc2, uint32_t nLeaf, uint32_t gap)
{
    uint64_t bottle_rate, bottle_delay;
    uint64_t access_rate, access_delay;

    getInstance(instance, bottle_rate, access_rate, bottle_delay, access_delay);
    //auto bufSize = std::max<uint32_t>(DEFAULT_PACKET_SIZE, bottle_rate / 8000 *  bottle_delay);
    //int BDP = bufSize / DEFAULT_PACKET_SIZE;

    // 输出各项配置
    NS_LOG_UNCOND("bottle_rate: " << bottle_rate / 1e6 << "Mbps");
    NS_LOG_UNCOND("access_rate: " << access_rate / 1e6 << "Mbps");
    NS_LOG_UNCOND("RTT: " << bottle_delay << "ms");
    // NS_LOG_UNCOND("BDP: " << BDP);
    NS_LOG_UNCOND("cc1: " << cc1);
    NS_LOG_UNCOND("cc1: " << cc2);

    PointToPointHelper bottleNeckLink;
    bottleNeckLink.SetDeviceAttribute("DataRate", DataRateValue(DataRate(bottle_rate)));
    bottleNeckLink.SetChannelAttribute("Delay", TimeValue(MilliSeconds(bottle_delay / 2)));
    bottleNeckLink.SetQueue("ns3::DropTailQueue", "MaxSize", StringValue("20p"));

    PointToPointHelper pointToPointLeaf;
    pointToPointLeaf.SetDeviceAttribute("DataRate", DataRateValue(DataRate(access_rate)));
    pointToPointLeaf.SetChannelAttribute("Delay", TimeValue(MilliSeconds(access_delay)));
    pointToPointLeaf.SetQueue("ns3::DropTailQueue", "MaxSize", StringValue("20p"));

    NodeContainer routers, leftLeaf, rightLeaf;
    NetDeviceContainer routerDevices, 
            leftLeafDevices, rightLeafDevices, 
            leftRouterDevices, rightRouterDevices;

    routers.Create(2);
    leftLeaf.Create(nLeaf);
    rightLeaf.Create(nLeaf);

    // Install Stack
    InternetStackHelper stack;
    stack.Install(routers);
    stack.Install(leftLeaf);
    stack.Install(rightLeaf);

    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        NetDeviceContainer c = pointToPointLeaf.Install(routers.Get(0), leftLeaf.Get(i));
        leftRouterDevices.Add(c.Get(0));
        leftLeafDevices.Add(c.Get(1));
    }
    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        NetDeviceContainer c = pointToPointLeaf.Install(routers.Get(1), rightLeaf.Get(i));
        rightRouterDevices.Add(c.Get(0));
        rightLeafDevices.Add(c.Get(1));
    }

    routerDevices = bottleNeckLink.Install(routers);

    TrafficControlHelper tchFq;
    tchFq.SetRootQueueDisc("ns3::FqCoDelQueueDisc");
    tchFq.SetQueueLimits("ns3::DynamicQueueLimits", "HoldTime", StringValue("1ms"));
    tchFq.Install (leftRouterDevices);
    tchFq.Install (leftLeafDevices);
    tchFq.Install (rightRouterDevices);
    tchFq.Install (rightLeafDevices);
    tchFq.Install (routerDevices);

    if(trigger_loss){
            trigger_loss->RegisterDevice(routerDevices.Get(1));
    }

    Ipv4AddressHelper leftIp("10.1.1.0", "255.255.255.0");
    Ipv4AddressHelper rightIp("10.2.1.0", "255.255.255.0");
    Ipv4AddressHelper routerIp("10.3.1.0", "255.255.255.0");

    vector<Ipv4Address> server_addr;
    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        NetDeviceContainer ndc;
        ndc.Add(leftRouterDevices.Get(i));
        ndc.Add(leftLeafDevices.Get(i));
        leftIp.Assign(ndc);
        leftIp.NewNetwork();
    }

    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        NetDeviceContainer ndc;
        Ipv4InterfaceContainer ifc;
        ndc.Add(rightRouterDevices.Get(i));
        ndc.Add(rightLeafDevices.Get(i));
        ifc = rightIp.Assign(ndc);
        server_addr.push_back(ifc.GetAddress(1));
        rightIp.NewNetwork();
    }
    routerIp.Assign(routerDevices);

    Ipv4GlobalRoutingHelper::PopulateRoutingTables ();

    uint32_t app_start = 5;
    uint32_t app_stop = 100;

    uint16_t server_port = 5000;
    for(uint32_t i = 0; i < nLeaf; i ++ )
    {
        Ptr<Node> client = leftLeaf.Get(i);

        Ptr<TcpL4Protocol> proto;
        proto = client->GetObject<TcpL4Protocol>();
        proto->SetAttribute("SocketType", TypeIdValue(TcpCubic::GetTypeId()));

        ApplicationContainer clientApp;
        BulkSendHelper tcp ("ns3::TcpSocketFactory", Address ());
        tcp.SetAttribute ("MaxBytes", UintegerValue (7500000000));
        InetSocketAddress firstDestAddress (server_addr[i], server_port);
        cout << server_addr[i] << endl;
        tcp.SetAttribute ("Remote", AddressValue (firstDestAddress));
        clientApp = tcp.Install(client);
        clientApp.Start (Seconds (app_start + i * gap));
        clientApp.Stop (Seconds (app_stop));

        Ptr<Node> server = rightLeaf.Get(i);
        proto = server->GetObject<TcpL4Protocol>();
        proto->SetAttribute("SocketType", TypeIdValue(TcpCubic::GetTypeId()));
        Address sinkAddress (InetSocketAddress (Ipv4Address::GetAny (), server_port));
        ApplicationContainer sinkApp;
        PacketSinkHelper sinkHelper ("ns3::TcpSocketFactory", sinkAddress);
        sinkApp = sinkHelper.Install (server);
        sinkApp.Start (Seconds (app_start + i * gap));
        sinkApp.Stop (Seconds (app_stop));
    }

    Simulator::Schedule(Seconds(0.0), &monitor, Simulator::Now ().GetSeconds ());

    Simulator::Stop(Seconds(app_stop) + Seconds(20));
    Simulator::Run();
    Simulator::Destroy();
}

int main(int argc, char *argv[])
{
    std::string instance = std::string("1");
    std::string loss_str("0"); // config random loss
    std::string cc1("cubic");
    std::string cc2("cubic");
    std::string data_folder("no-one");
    uint32_t nLeaf = 4;
    uint32_t start_gap = 0;

    CommandLine cmd;
    cmd.AddValue("it", "instacne", instance);
    cmd.AddValue("folder", "folder name to collect data", data_folder);
    cmd.AddValue("lo", "loss", loss_str); // 10 means the dev will introduce 10/1000 % random loss
    cmd.AddValue("cc1", "congestion algorithm1", cc1);
    cmd.AddValue("cc2", "congestion algorithm2", cc2);
    cmd.AddValue("nLeaf", "number of leaf", nLeaf);
    cmd.AddValue("gap", "gap between application start", start_gap);
    cmd.Parse(argc, argv);

    Config::SetDefault ("ns3::UdpSocket::RcvBufSize",UintegerValue (100768000));

    int loss_integer = std::stoi(loss_str);
    double random_loss = loss_integer * 1.0 / 100000;
    std::unique_ptr<TriggerRandomLoss> triggerloss = nullptr;
    if (loss_integer > 0)
    {
        Config::SetDefault("ns3::RateErrorModel::ErrorRate", DoubleValue(random_loss));
        Config::SetDefault("ns3::RateErrorModel::ErrorUnit", StringValue("ERROR_UNIT_PACKET"));
        Config::SetDefault("ns3::BurstErrorModel::ErrorRate", DoubleValue(random_loss));
        Config::SetDefault("ns3::BurstErrorModel::BurstSize", StringValue("ns3::UniformRandomVariable[Min=1|Max=3]"));
        triggerloss.reset(new TriggerRandomLoss());         //reset是unique_ptr的函数，等同于赋值
        triggerloss->Start();
    }
    NS_LOG_UNCOND("random_loss: " << random_loss * 100 << "%");

    // 判断拥塞控制算法输入合法性
    if (0 == cc1.compare("reno") || 0 == cc1.compare("cubic") ||
        0 == cc1.compare("bbr") || 0 == cc1.compare("bbrv2") ||
        0 == cc1.compare("copa") || 0 == cc1.compare("vegas"))
    {
    }
    else
    {
        NS_ASSERT_MSG(0, "please input correct cc1");
    }
    if (0 == cc2.compare("reno") || 0 == cc2.compare("cubic") ||
        0 == cc2.compare("bbr") || 0 == cc2.compare("bbrv2") ||
        0 == cc2.compare("copa") || 0 == cc2.compare("vegas"))
    {
    }
    else
    {
        NS_ASSERT_MSG(0, "please input correct cc2");
    }

    // 检查QUICHE依赖
    const char *envKey = "QUICHE_SRC_DIR";
    char *envValue = getenv(envKey);
    if (envValue)
        NS_LOG_UNCOND("QUICHE_SRC_DIR:" << envValue);

    // 创建结果目录文件夹
    char buffer[128] = {0};
    if (getcwd(buffer, sizeof(buffer)) != buffer)
    {
        NS_LOG_ERROR("path error");
        return -1;
    }
    std::string ns3_path(buffer, ::strlen(buffer));
    if ('/' != ns3_path.back())
    {
        ns3_path.push_back('/');
    }
    std::string trace_folder = ns3_path + "traces/";
    NS_LOG_UNCOND("trace_folder: " << trace_folder);
    if (!MakePath(trace_folder))
    {
        std::cout << trace_folder << " is not right" << std::endl;
        return -1;
    }

    std::string log_path = trace_folder + data_folder + "/";
    set_quic_trace_folder(log_path);
    if (!MakePath(log_path))
    {
        std::cout << log_path << " is not right" << std::endl;
        return -1;
    }
    NS_LOG_UNCOND("log_path: " << log_path);

    set_quic_log_error_level(); // print quiche lib error
    set_quic_goodput_interval(MilliSeconds(200));
    quic::ContentInit('a');     // QUIC协议数据负载内容（全部填写a）

    std::string quic_cert_path = std::string(envValue) + "utils/data/quic-cert/";
    if (ns3::set_quic_cert_key(quic_cert_path))
    {
        quic::BackendType type = quic::BackendType::BANDWIDTH;

        QuicClientTraceType client_log_flag = E_QC_IN_FLIGHT | E_QC_SEND_RATE;
        QuicServerTraceType server_log_flag = E_QS_OWD | E_QS_LOST | E_QS_GOODPUT;

        RegisterExternalCongestionFactory();

        test_app_on_dumbbell(
            instance, // 链路属性
            client_log_flag, // Client负责接收数据，追踪inflight，发送速率
            server_log_flag, // Server负责接收数据，追踪单向时延，丢包率，接受速率
            type,
            triggerloss.get(),
            cc1,
            cc2,
            nLeaf,
            start_gap);
    }
    return 0;
}

[SoonyangZhang]

add the following code 1 ns3-quic-channel.h

void config_channel_params(int num_pkts, int interval_ms);  

2 ns3-quic-channel.cc

void config_channel_params(int num_pkts, int interval_ms) {
  kWritePackets = num_pkts;
  kWriteDelta = QuicTime::Delta::FromMilliseconds(interval_ms);
}

3 ns3-quic-util.h

void set_quic_stream_buffer_threshold(uint64_t threshold, int num_pkt, int interval_ms);  

4 ns3-quic-util.cc (of course, add header info: #include )

void set_quic_stream_buffer_threshold(uint64_t threshold, int num_pkt, int interval_ms) {
  SetQuicFlag(FLAGS_quic_buffered_data_threshold, threshold);
  quic::config_channel_params(num_pkt, interval_ms);
  //std::cout<<GetQuicFlag(FLAGS_quic_buffered_data_threshold)<<std::endl;
}

5 quic-main.cc (https://github.com/SoonyangZhang/quic-on-ns3/blob/main/scratch/quic-main.cc#L542)

set_quic_stream_buffer_threshold(1000*1024, 100, 5);

For 100Mbps, these params works.

[MCxiaofang]

it works!