Thread Priority Setting Issues in ThreadPool.h

[luohaha66]

Thread Priority Setting Issue in ThreadPool.h

Code Related to Setting Thread Pool Priority

    static bool setPriority(Priority priority = PRIORITY_NORMAL, std::thread::native_handle_type threadId = 0) {
        // set priority
#if defined(_WIN32)
        static int Priorities[] = { THREAD_PRIORITY_LOWEST, THREAD_PRIORITY_BELOW_NORMAL, THREAD_PRIORITY_NORMAL, THREAD_PRIORITY_ABOVE_NORMAL, THREAD_PRIORITY_HIGHEST };
        if (priority != PRIORITY_NORMAL && SetThreadPriority(GetCurrentThread(), Priorities[priority]) == 0) {
            return false;
        }
        return true;
#else
        static int Min = sched_get_priority_min(SCHED_OTHER);
        if (Min == -1) {
            return false;
        }
        static int Max = sched_get_priority_max(SCHED_OTHER);
        if (Max == -1) {
            return false;
        }
        static int Priorities[] = {Min, Min + (Max - Min) / 4, Min + (Max - Min) / 2, Min + (Max - Min) * 3 / 4, Max};

        if (threadId == 0) {
            threadId = pthread_self();
        }
        struct sched_param params;
        params.sched_priority = Priorities[priority];
        return pthread_setschedparam(threadId, SCHED_OTHER, &params) == 0;
#endif
    }

Existing Issues

• SCHED_OTHER is the default Linux timesharing scheduling policy, which does not support priority setting. Only real-time scheduling policies like SCHED_FIFO and SCHED_RR support thread priority setting.
• The Min and Max values in the above code are both 0. Regardless of the value of the priority parameter passed in by the caller, the value of params.sched_priority is always 0.

ThreadPool.h 中线程优先级设置问题

设置线程池优先级相关代码

    static bool setPriority(Priority priority = PRIORITY_NORMAL, std::thread::native_handle_type threadId = 0) {
        // set priority
#if defined(_WIN32)
        static int Priorities[] = { THREAD_PRIORITY_LOWEST, THREAD_PRIORITY_BELOW_NORMAL, THREAD_PRIORITY_NORMAL, THREAD_PRIORITY_ABOVE_NORMAL, THREAD_PRIORITY_HIGHEST };
        if (priority != PRIORITY_NORMAL && SetThreadPriority(GetCurrentThread(), Priorities[priority]) == 0) {
            return false;
        }
        return true;
#else
        static int Min = sched_get_priority_min(SCHED_OTHER);
        if (Min == -1) {
            return false;
        }
        static int Max = sched_get_priority_max(SCHED_OTHER);
        if (Max == -1) {
            return false;
        }
        static int Priorities[] = {Min, Min + (Max - Min) / 4, Min + (Max - Min) / 2, Min + (Max - Min) * 3 / 4, Max};

        if (threadId == 0) {
            threadId = pthread_self();
        }
        struct sched_param params;
        params.sched_priority = Priorities[priority];
        return pthread_setschedparam(threadId, SCHED_OTHER, &params) == 0;
#endif
    }

存在问题

• SCHED_OTHER 为默认的 Linux 分时调度策略，并不支持优先级的设置，只有 SCHED_FIFO、SCHED_RR 这样的实时调度策略，才支持线程优先级的设置。
• 上述代码中的 Min 以及 Max 其值皆为 0，无论调用者传入的参数 priority 为何值，params.sched_priority 的值都为 0。

TRANS_BY_GITHUB_AI_ASSISTANT

[xia-chu]

No, I'm debugging on a Mac, and it should be similar on Linux:

并不是吧 我在mac上调试是这样的，linux应该也类似：

TRANS_BY_GITHUB_AI_ASSISTANT

[xia-chu]

This code is actually copied from other projects.

这个代码其实我也是从其他项目扣得

TRANS_BY_GITHUB_AI_ASSISTANT

[luohaha66]

man sched Explanation of SCHED_OTHER

• SCHED_OTHER: Default Linux time-sharing scheduling SCHED_OTHER can be used at only static priority 0 (i.e., threads under real-time policies always have priority over SCHED_OTHER processes). SCHED_OTHER is the standard Linux time-sharing scheduler that is intended for all threads that do not require the special real-time mechanisms

Test Code

#include <stdio.h>
#include <sched.h>

int main(void)
{
    int Min = sched_get_priority_min(SCHED_OTHER);
    int Max = sched_get_priority_max(SCHED_OTHER);
    // 输出为 0 0
    printf("min %d, max %d\n", Min, Max);

    Min = sched_get_priority_min(SCHED_IDLE);
    Max = sched_get_priority_max(SCHED_IDLE);
    // 输出为 0 0
    printf("min %d, max %d\n", Min, Max);

    Min = sched_get_priority_min(SCHED_BATCH);
    Max = sched_get_priority_max(SCHED_BATCH);
    // 输出为 0 0
    printf("min %d, max %d\n", Min, Max);

    Min = sched_get_priority_min(SCHED_FIFO);
    Max = sched_get_priority_max(SCHED_FIFO);
    // 输出为 1 99
    printf("min %d, max %d\n", Min, Max);

    Min = sched_get_priority_min(SCHED_RR);
    Max = sched_get_priority_max(SCHED_RR);
    // 输出为 1 99
    printf("min %d, max %d\n", Min, Max);
    return 0;
}

Test Platform

uname -a Output	cpu	gcc Version
Linux 5.15.0-86-generic Ubuntu x86_64 GNU/Linux	AMD Ryzen 7 3800X	11.3
Linux 5.10.160 SMP aarch64 GNU/Linux	RK3588	10.3
Linux 4.19.111 SMP PREEMPT armv7l GNU/Linux	RV1126	8.3

Test Results

• I don't have a Mac environment, so I only tested on Linux.
• I tested on the 3 platforms above, and the output is the same.
• From the results, Linux and Mac are really different.

man sched 对 SCHED_OTHER 的解释

• SCHED_OTHER: Default Linux time-sharing scheduling SCHED_OTHER can be used at only static priority 0 (i.e., threads under real-time policies always have priority over SCHED_OTHER processes). SCHED_OTHER is the standard Linux time-sharing scheduler that is intended for all threads that do not require the special real-time mechanisms

测试代码

#include <stdio.h>
#include <sched.h>

int main(void)
{
    int Min = sched_get_priority_min(SCHED_OTHER);
    int Max = sched_get_priority_max(SCHED_OTHER);
    // 输出为 0 0
    printf("min %d, max %d\n", Min, Max);

    Min = sched_get_priority_min(SCHED_IDLE);
    Max = sched_get_priority_max(SCHED_IDLE);
    // 输出为 0 0
    printf("min %d, max %d\n", Min, Max);

    Min = sched_get_priority_min(SCHED_BATCH);
    Max = sched_get_priority_max(SCHED_BATCH);
    // 输出为 0 0
    printf("min %d, max %d\n", Min, Max);

    Min = sched_get_priority_min(SCHED_FIFO);
    Max = sched_get_priority_max(SCHED_FIFO);
    // 输出为 1 99
    printf("min %d, max %d\n", Min, Max);

    Min = sched_get_priority_min(SCHED_RR);
    Max = sched_get_priority_max(SCHED_RR);
    // 输出为 1 99
    printf("min %d, max %d\n", Min, Max);
    return 0;
}

测试平台

uname -a 输出	cpu	gcc 版本
Linux 5.15.0-86-generic Ubuntu x86_64 GNU/Linux	AMD Ryzen 7 3800X	11.3
Linux 5.10.160 SMP aarch64 GNU/Linux	RK3588	10.3
Linux 4.19.111 SMP PREEMPT armv7l GNU/Linux	RV1126	8.3

测试结果

• 我身边没有 Mac 相关环境，就只在 linux 上测试了。
• 一共测了上面 3 个平台，输出都是一样的。
• 从结果来看，linux 和 mac 还真不一样。

TRANS_BY_GITHUB_AI_ASSISTANT

[xia-chu]

@luohaha66 What is your suggestion for modification? Should we switch to SCHED_RR or SCHED_FIFO?

@luohaha66 那您的建议是怎么修改？换SCHED_RR还是SCHED_FIFO？

TRANS_BY_GITHUB_AI_ASSISTANT

[luohaha66]

@luohaha66 那您的建议是怎么修改？换SCHED_RR还是SCHED_FIFO？

I will reply to you during work hours tomorrow.

@luohaha66 那您的建议是怎么修改？换SCHED_RR还是SCHED_FIFO？

我明天工作时间回复您。

TRANS_BY_GITHUB_AI_ASSISTANT

[luohaha66]

How to Choose Between SCHED_RR and SCHED_FIFO

SCHED_FIFO

• First-In, First-Out scheduling.
• Can only be used with a static priority higher than 0, meaning that when a SCHED_FIFO thread becomes runnable, it will always immediately preempt any currently running SCHED_OTHER, SCHED_BATCH, or SCHED_IDLE thread.
• A SCHED_FIFO thread runs continuously until it is blocked by an I/O request, preempted by a higher priority thread, or calls sched_yield.
• SCHED_FIFO is a simple scheduling algorithm with no time slicing. The following rules apply to threads scheduled under the SCHED_FIFO policy:
  • A running SCHED_FIFO thread that is preempted by another higher priority thread will remain at the head of its priority list and will resume execution immediately when all higher priority threads are blocked again.
  • When a blocked SCHED_FIFO thread becomes runnable, it will be inserted at the end of the priority list.

SCHED_RR

• Round-Robin scheduling.
• SCHED_RR is a simple enhancement of SCHED_FIFO. Everything described above for SCHED_FIFO also applies to SCHED_RR, except that each thread is only allowed to run for a maximum time slice.
• If a SCHED_RR thread has run for a period equal to or greater than the time slice, it will be placed at the end of the priority list.
• A SCHED_RR thread that is preempted by a higher priority thread and subsequently resumes execution as the running thread will complete the unexpired portion of its time slice.

Analysis

• Assume that there are the following 3 threads on cpu0:

  Thread Name	Priority	Scheduling Policy
  t0	high	FIFO or RR
  t1	mid	FIFO or RR
  t3	0	OTHER

• Assuming that t0 and t1 use the FIFO policy, as long as the t0 thread is ready, it will immediately preempt t1 and t3 until t0 voluntarily relinquishes the cpu. Only then will t1 or t3 be run.

• Assuming that t0 and t1 use the RR policy, the difference from FIFO is that after the t0 thread uses up its time slice, a scheduling event will occur. At this time, t0 is still the highest priority thread on cpu0, and it will get to run again. In other words, as long as t0 does not voluntarily relinquish the cpu, it will continue to run. This is consistent with FIFO, but with additional overhead.

• Assuming that t0 and t1 have the same priority, if t0 runs first under the FIFO policy, then t1 will only get to run after t0 voluntarily relinquishes the cpu. Under the RR policy, if t0 runs first, t1 will get to run after t0's time slice is consumed.

SCHED_FIFO Use Cases

• In the case of setting cpu affinity, if there is only one thread with the same priority on the same core, then SCHED_FIFO should be used.
• In the case of not setting cpu affinity, if the number of threads with the same priority is less than or equal to the total number of cpu cores, then SCHED_FIFO should be used.

SCHED_RR Use Cases

• In the case of setting cpu affinity, if more than one thread with the same priority needs to be bound to the same core, then SCHED_RR should be used.
• In the case of not setting cpu affinity, if the number of threads with the same priority exceeds the total number of cpu cores, then SCHED_RR should be used.

Recommendation

• The zlmediakit thread pool usually only creates a number of threads equal to the number of cpu cores, and each cpu core is only bound to one thread with the same priority. It is recommended to use SCHED_FIFO.

SCHED_RR SCHED_FIFO 如何选择

SCHED_FIFO

• 先进先出调度。
• 只能在静态优先级高于0的情况下使用，这意味着当 SCHED_FIFO 线程变得可运行时，它总是立即抢占当前正在运行的任何 SCHED_OTHER、SCHED_BATCH 或 SCHED_IDLE 线程。
• SCHED_FIFO 线程一直运行到被 I/O 请求阻塞、被高优先级线程抢占或调用 sched_yield 为止。
• SCHED_FIFO 是一种简单的调度算法，没有时间切片。对于 SCHED_FIFO 策略下调度的线程，应用以下规则:
  • 被另一个高优先级线程抢占的正在运行的 SCHED_FIFO 线程将保持在其优先级列表的头部，并在所有高优先级线程再次阻塞时立即恢复执行。
  • 当阻塞的 SCHED_FIFO 线程变为可运行时，它将被插入到优先级列表的末尾。

SCHED_RR

• 循环调度。
• SCHED_RR 是 SCHED_FIFO 的简单增强。上面为 SCHED_FIFO 描述的所有内容也适用于 SCHED_RR，除了每个线程只允许在最大时间片内运行。
• 如果 SCHED_RR 线程已经运行了等于或大于该时间片的时间段，那么它将被放在优先级列表的末尾。
• 被高优先级线程抢占并随后作为运行线程恢复执行的 SCHED_RR 线程将完成其循环时间量的未过期部分。

分析

• 假设在 cpu0 上存在以下3个线程：

  线程名	优先级	调度策略
  t0	high	FIFO 或 RR
  t1	mid	FIFO 或 RR
  t3	0	OTHER

• 假设 t0 和 t1 使用 FIFO 策略，则只要 t0 线程就绪，就会立刻抢占 t1 和 t3，直到 t0 主动放弃 cpu 为止，t1 或者 t3 才会被运行。

• 假设 t0 和 t1 使用 RR 策略，与 FIFO 的区别就是，t0 线程在使用完时间片以后，会发生一次调度， 此时 t0 仍然是 cpu0 上优先级最高的线程，其会再次获得运行。也就是说，只要 t0 不主动放弃 cpu，其会一直运行。和 FIFO 一致，但是多了额外的开销。

• 假设 t0 和 t1 优先级一致，在使用 FIFO 策略的情况下，如果 t0 先运行，那么 t1 只有等到 t0 主动放弃 cpu，才会得到运行。在使用 RR 策略的情况下，如果 t0 先运行，t1 在 t0 时间片消耗完之后便可得到运行。

SCHED_FIFO 适用情况

• 在设置 cpu 亲和性的情况下，如果在同一个核心上只存在一个同优先级线程，则应使用 SCHED_FIFO。
• 在不设置 cpu 亲和性的情况下，如果同优先级线程数小于或等于 cpu 总核心数，则应使用 SCHED_FIFO。

SCHED_RR 适用情况

• 在设置 cpu 亲和性的情况下，如果在同一个核心上需要绑定一个以上的同优先级线程，则应使用 SCHED_RR。
• 在不设置 cpu 亲和性的情况下，如果同优先级线程数超过 cpu 总核心数，则应使用 SCHED_RR。

建议

• zlmediakit 线程池通常只会创建与 cpu 核心数相等的线程数，并且每个 cpu 核心都只绑定了一个同优先级的线程，推荐使用 SCHED_FIFO 。

TRANS_BY_GITHUB_AI_ASSISTANT

[xia-chu]

Good! Let's modify it like this:

Index: src/Thread/ThreadPool.h
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/Thread/ThreadPool.h b/src/Thread/ThreadPool.h
--- a/src/Thread/ThreadPool.h   (revision 3fd2b856b6856dd679a32c673431561c0affdd0c)
+++ b/src/Thread/ThreadPool.h   (date 1698292460548)
@@ -86,11 +86,11 @@
         }
         return true;
 #else
-        static int Min = sched_get_priority_min(SCHED_OTHER);
+        static int Min = sched_get_priority_min(SCHED_FIFO);
         if (Min == -1) {
             return false;
         }
-        static int Max = sched_get_priority_max(SCHED_OTHER);
+        static int Max = sched_get_priority_max(SCHED_FIFO);
         if (Max == -1) {
             return false;
         }
@@ -101,7 +101,7 @@
         }
         struct sched_param params;
         params.sched_priority = Priorities[priority];
-        return pthread_setschedparam(threadId, SCHED_OTHER, &params) == 0;
+        return pthread_setschedparam(threadId, SCHED_FIFO, &params) == 0;
 #endif
     }

善哉！那这样修改吧：

Index: src/Thread/ThreadPool.h
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/Thread/ThreadPool.h b/src/Thread/ThreadPool.h
--- a/src/Thread/ThreadPool.h   (revision 3fd2b856b6856dd679a32c673431561c0affdd0c)
+++ b/src/Thread/ThreadPool.h   (date 1698292460548)
@@ -86,11 +86,11 @@
}
return true;
#else
-        static int Min = sched_get_priority_min(SCHED_OTHER);
+        static int Min = sched_get_priority_min(SCHED_FIFO);
if (Min == -1) {
return false;
}
-        static int Max = sched_get_priority_max(SCHED_OTHER);
+        static int Max = sched_get_priority_max(SCHED_FIFO);
if (Max == -1) {
return false;
}
@@ -101,7 +101,7 @@
}
struct sched_param params;
params.sched_priority = Priorities[priority];
-        return pthread_setschedparam(threadId, SCHED_OTHER, &params) == 0;
+        return pthread_setschedparam(threadId, SCHED_FIFO, &params) == 0;
#endif
}

`TRANS_BY_GITHUB_AI_ASSISTANT`

[luohaha66]

Thank you, the above modifications are sufficient.

谢谢，按照上述修改即可。

TRANS_BY_GITHUB_AI_ASSISTANT

[xia-chu]

Submitted. Thanks!

已提交 感谢！

TRANS_BY_GITHUB_AI_ASSISTANT