lfeng14
commented
2 months ago 复现测试demo
#include <stdio.h>
typedef long long ulonglong;
static inline ulonglong my_timer_cycles(void)
{
# if __has_builtin(__builtin_readcyclecounter) && !defined (__aarch64__)
return __builtin_readcyclecounter();
# elif defined _WIN32 || defined __i386__ || defined __x86_64__
return __rdtsc();
# elif defined(__INTEL_COMPILER) && defined(__ia64__) && defined(HAVE_IA64INTRIN_H)
return (ulonglong) __getReg(_IA64_REG_AR_ITC); /* (3116) */
#elif defined(__GNUC__) && defined(__ia64__)
{
ulonglong result;
__asm __volatile__ ("mov %0=ar.itc" : "=r" (result));
return result;
}
#elif defined __GNUC__ && defined __powerpc__
return __builtin_ppc_get_timebase();
#elif defined(__GNUC__) && defined(__sparcv9) && defined(_LP64)
{
ulonglong result;
__asm __volatile__ ("rd %%tick,%0" : "=r" (result));
return result;
}
#elif defined(__GNUC__) && defined(__sparc__) && !defined(_LP64)
{
union {
ulonglong wholeresult;
struct {
ulong high;
ulong low;
} splitresult;
} result;
__asm __volatile__ ("rd %%tick,%1; srlx %1,32,%0" : "=r" (result.splitresult.high), "=r" (result.splitresult.low));
return result.wholeresult;
}
#elif defined(__GNUC__) && defined(__s390__)
/* covers both s390 and s390x */
{
ulonglong result;
__asm__ __volatile__ ("stck %0" : "=Q" (result) : : "cc");
return result;
}
#elif defined(__GNUC__) && defined (__aarch64__)
{
ulonglong result;
__asm __volatile("mrs %0, CNTVCT_EL0" : "=&r" (result));
return result;
}
#elif defined(HAVE_SYS_TIMES_H) && defined(HAVE_GETHRTIME)
/* gethrtime may appear as either cycle or nanosecond counter */
printf("a\n");
return (ulonglong) gethrtime();
#else
printf("b\n");
return 0;
#endif
}
int main() {
unsigned long cycles;
// 内嵌汇编指令，读取周期计数器
// asm volatile ("rdcycle %0" : "=r" (cycles));
// __builtin_readcyclecounter();
cycles = my_timer_cycles();
printf("Cycle count: %lu\n", cycles);
return 0;
}
现象
在RISCV机器上，使用llvm构建mariadb后，启动未成功。
journalctl日志
软件包日志
如何找到哪个指令导致的illegal instruction
Thread 1 "mysqld" received signal SIGILL, Illegal instruction. rw_lock_x_lock_wait_func (lock=0x2aacf707f0, threshold=0, file_name=0x2aab8fa978 "/home/lkp/rpmbuild/BUILD/mariadb-10.5.24/storage/innobase/fsp/fsp0fsp.cc", line=559) at /usr/src/debug/mariadb-10.5.24-1.oe2403.riscv64/storage/innobase/sync/sync0rw.cc:465 465 rw_lock_stats.rw_x_spin_round_count.add(n_spins); Missing separate debuginfos, use: dnf debuginfo-install Judy-1.0.5-19.mg2403.riscv64 libaio-0.3.113-9.mg2403.riscv64 libcap-2.69-3.mg2403.riscv64 libgcc-12.3.1-25.mg2403.riscv64 libgcrypt-1.10.2-1.mg2403.riscv64 libgpg-error-1.47-1.mg2403.riscv64 libstdc++-12.3.1-25.mg2403.riscv64 libxcrypt-4.4.36-2.mg2403.riscv64 lz4-1.9.4-2.mg2403.riscv64 openssl-libs-3.0.12-3.oe2403.riscv64 pcre2-10.42-6.mg2403.riscv64 sssd-client-2.9.4-2.mg2403.riscv64 systemd-libs-255-10.mg2403.riscv64 xz-libs-5.4.4-1.mg2403.riscv64 zlib-1.3.1-1.mg2403.riscv64 (gdb) bt
0 rw_lock_x_lock_wait_func (lock=0x2aacf707f0, threshold=0,
1 0x0000002aab6941fc in rw_lock_x_lock_low (lock=0x2aacf707f0, pass=,
2 0x0000002aab693ece in rw_lock_x_lock_func (lock=0x2aacf707f0, pass=0,
3 0x0000002aab733e16 in pfs_rw_lock_x_lock_func (lock=0x2aacf707f0, pass=0, line=559,
4 mtr_t::x_lock_space (this=0x3fffffcbd8, space=0x2aacf70768, line=559, file=)
5 fsp_header_init (space=0x2aacf70768, size=768, mtr=0x3fffffcbd8)
6 0x0000002aab68d44a in srv_start (create_new_db=true)
7 0x0000002aab5ca81a in innodb_init (p=)
8 0x0000002aab35be50 in ha_initialize_handlerton (plugin=0x2aac994110)
9 0x0000002aab1bfd88 in plugin_do_initialize (plugin=0x2aac994110, state=@0x3fffffd39c: 4)
10 0x0000002aab1bf9b6 in plugin_initialize (tmp_root=0x3fffffd950, plugin=0x2aac994110,
11 0x0000002aab1bf6e2 in plugin_init (argc=0x2aac0492f0, argv=0x2aac952ee0,
12 0x0000002aab10ee10 in init_server_components ()
13 0x0000002aab10bb0a in mysqld_main (argc=55, argv=0x2aac952ee0)
14 0x0000003ff7bd1958 in ?? () from /usr/lib64/libc.so.6
15 0x0000003ff7bd1a00 in __libc_start_main () from /usr/lib64/libc.so.6
16 0x0000002aab108f14 in _start ()
// UNIV_INLINE void rw_lock_x_lock_wait_func( /=====================/ rw_lock_t lock, /!< in: pointer to rw-lock */
ifdef UNIV_DEBUG
endif
{ HMT_low(); HMT_medium(); rw_lock_stats.rw_x_spin_round_count.add(n_spins); <----
}
static inline ulonglong my_timer_cycles(void) {
if has_builtin(__builtin_readcyclecounter) && !defined (aarch64) === llvm支持builtin_readcyclecounter
return __builtin_readcyclecounter();
elif defined _WIN32 || defined i386 || defined __x86_64__
return __rdtsc();
elif defined(__INTEL_COMPILER) && defined(ia64) && defined(HAVE_IA64INTRIN_H)
return (ulonglong) __getReg(_IA64_REG_AR_ITC); / (3116) /
elif defined(GNUC) && defined(ia64)
{ ulonglong result; asm volatile__ ("mov %0=ar.itc" : "=r" (result)); return result; }
elif defined GNUC && defined powerpc
return __builtin_ppc_get_timebase();
elif defined(GNUC) && defined(__sparcv9) && defined(_LP64)
{ ulonglong result; asm volatile__ ("rd %%tick,%0" : "=r" (result)); return result; }
elif defined(GNUC) && defined(sparc) && !defined(_LP64)
{ union { ulonglong wholeresult; struct { ulong high; ulong low; } splitresult; } result; asm volatile__ ("rd %%tick,%1; srlx %1,32,%0" : "=r" (result.splitresult.high), "=r" (result.splitresult.low)); return result.wholeresult; }
elif defined(GNUC) && defined(s390)
/ covers both s390 and s390x / { ulonglong result; asm volatile ("stck %0" : "=Q" (result) : : "cc"); return result; }
elif defined(GNUC) && defined (aarch64)
{ ulonglong result; asm volatile("mrs %0, CNTVCT_EL0" : "=&r" (result)); return result; }
elif defined(HAVE_SYS_TIMES_H) && defined(HAVE_GETHRTIME)
/ gethrtime may appear as either cycle or nanosecond counter / return (ulonglong) gethrtime();
else
return 0; =============== gcc未使用rdcycle指令，直接返回0，这里是不对的。
endif
}
We used to unconditionnally expose the cycle and instret csrs to userspace, which gives rise to security concerns.
So now we only allow access to hw counters from userspace through the perf framework which will handle context switches, per-task events...etc. A sysctl allows to revert the behaviour to the legacy mode so that userspace applications which are not ready for this change do not break.
But the default value is to allow userspace only through perf: this will break userspace applications which rely on direct access to rdcycle. This choice was made for security reasons [1][2]: most of the applications which use rdcycle can instead use rdtime to count the elapsed time.
解决方式
riscv架构上，将rdcycle指令切换为不受管控的rdtime指令，网上有相同案例请参考文末。
影响范围
软件包自身调用readcyclecounter指令，同时kernel 默认不使能该类指令，编译器又支持readcyclecounter后端指令时，这种情况会触发非法指令报错。
参考