Systemtap的使用

[BruceChen7]

资源

• https://linux.die.net/man/3/stapprobes
• http://www.slideshare.net/posullivan/monitoring-mysql-with-dtracesystemtap
• tapset
• brendan使用
• 语言
• 霸爷的使用心得
• 阿里的使用
• cloudflare的使用
• systemtap内核模块的编写使用

安装

 sudo apt-get install linux-headers-`uname -r`
 # 安装调试符号
 sudo apt-get install linux-image-`uname -r`-dbg
 可以在这个连接处找，然后用dpkg -i 来安装
 // http://ddebs.ubuntu.com/ubuntu/ubuntu/pool/main/l/linux/

查看某一行的值

brookchen@brookchen:~$ sudo stap -e 'probe process("/home/brookchen/test").statement("add@/home/brookchen/test.c:5") {printf("%d\n", $b)}' -c "./test"
hello world, 1
hello world3
3

探测点

kernel.function(PATTERN)
kernel.function(PATTERN).call
kernel.function(PATTERN).return
kernel.function(PATTERN).return.maxactive(VALUE)
kernel.function(PATTERN).inline
kernel.function(PATTERN).label(LPATTERN)
module(MPATTERN).function(PATTERN)
module(MPATTERN).function(PATTERN).call
module(MPATTERN).function(PATTERN).return.maxactive(VALUE)
module(MPATTERN).function(PATTERN).inline
kernel.statement(PATTERN)
kernel.statement(ADDRESS).absolute
module(MPATTERN).statement(PATTERN)
process(PROCESSPATH).function(PATTERN)
process(PROCESSPATH).function(PATTERN).call
process(PROCESSPATH).function(PATTERN).return
process(PROCESSPATH).function(PATTERN).inline
process(PROCESSPATH).statement(PATTERN)

brookchen@brookchen:~$ sudo stap -l "netdev.*"
netdev.change_mac
netdev.change_mtu
netdev.change_rx_flag
netdev.close
netdev.get_stats
netdev.hard_transmit
netdev.ioctl
netdev.open
netdev.receive
netdev.register
netdev.rx
netdev.set_promiscuity
netdev.transmit
netdev.unregister

sudo stap -d /usr/local/bin/redis-server -l 'process("redis-server").function("*")'
process("/usr/local/bin/redis-server").function("streamNew@/home/brookchen/Redis/redis-unstable/src/t_stream.c:52")
process("/usr/local/bin/redis-server").function("streamNextID@/home/brookchen/Redis/redis-unstable/src/t_stream.c:74")
process("/usr/local/bin/redis-server").function("streamParseIDOrReply@/home/brookchen/Redis/redis-unstable/src/t_stream.c:1143")

sudo stap -l "scheduler.*"

scheduler.balance
scheduler.cpu_off
scheduler.cpu_on
scheduler.ctxswitch
scheduler.kthread_stop
scheduler.migrate
scheduler.process_exit
scheduler.process_fork
scheduler.process_free
scheduler.process_wait
scheduler.signal_send
scheduler.tick
scheduler.wait_task
scheduler.wakeup
scheduler.wakeup_new

sudo stap -L 'vfs.write'
vfs.write pathname:string dev:long devname:string ino:long name:string argstr:string
brookchen@brookchen:~$ sudo stap -L 'vfs.*'    
vfs.__add_to_page_cache $page:struct page* $mapping:struct address_space* $offset:long unsigned int $gfp_mask:gfp_t
vfs.__set_page_dirty_buffers dev:long devname:string ino:long index:long name:string argstr:string size:long units:string $page:struct page*
vfs.add_to_page_cache dev:long devname:string ino:long index:long nrpages:long size:long units:string name:string argstr:string $page:struct page* $mapping:struct address_space* $offset:long unsigned int $gfp_mask:gfp_t
vfs.buffer_migrate_page dev:long ino:long devname:string index:long name:string argstr:string size:long units:string $mapping:struct address_space* $newpage:struct page* $page:struct page* $mode:enum migrate_mode $rc:int
vfs.do_mpage_readpage dev:long devname:string ino:long index:long size:long name:string argstr:string units:string $args:struct mpage_readpage_args* $blocks:sector_t[]
vfs.do_sync_read dev:long devname:string ino:long file:long pathname:string len:long pos:long buf:long name:string argstr:string size:long units:string bytes_to_read:long $file:struct file* $buf:char* $count:size_t $pos:loff_t*
vfs.do_sync_write dev:long devname:string ino:long file:long pathname:string len:long pos:long buf:long bytes_to_write:long name:string argstr:string size:long units:string $file:struct file* $p:char const* $count:size_t $pos:loff_t*
vfs.open name:string cred:unknown pathname:string argstr:string
vfs.read pathname:string dev:long devname:string ino:long name:string argstr:string
vfs.readv file:long pathname:string dev:long devname:string ino:long pos:long vec:long vlen:long bytes_to_read:long name:string argstr:string $file:struct file* $vec:struct iovec const* $vlen:long unsigned int $pos:loff_t* $flags:rwf_t $iovstack:struct iovec[] $iter:struct iov_iter $ret:ssize_t
vfs.remove_from_page_cache dev:long devname:string ino:long index:long name:string argstr:string $page:struct page* $shadow:void*
vfs.write pathname:string dev:long devname:string ino:long name:string argstr:string
vfs.writev file:long pathname:string dev:long devname:string ino:long pos:long vlen:long vec:long bytes_to_write:long name:string argstr:string $file:struct file* $vec:struct iovec const* $vlen:long unsigned int $pos:loff_t* $flags:rwf_t $iovstack:struct iovec[] $iter:struct iov_iter $ret:ssize_t

统计

• Scalable accumulation with <<<
• Aggregates values only when read
• Extract numeric values with function-like operators @count(v) @sum(v) @min(v) @max(v) @avg(v)
• Extract histograms that may be printed, indexed, or iterated @hist_linear(v, start, stop, interval), @hist_log(v)

  关系运算符

• < > <= >= == != =~ !~

其中，The =~ and !~ operators 执行正则匹配，=~表示返回1，!~则表示相反。

in

• This expression evaluates to true if the array contains an element with the specified index.

Pointer typecasting

@cast(p, "type_name"[, "module"])->member

具体的使用：

@cast(pointer, "task_struct", "kernel")->parent->tgid

#!/usr/bin/stap
probe kernel.function("do_mmap_pgoff") {
       printf("%s(%d) %s (%s)\n",execname(),pid(),$pgoff,$len) // 函数的入参
}

#!/usr/bin/stap
probe kernel.statement("fb_mmap@drivers/video/fbmem.c:1357")
{
        printf("%s(%d) %x\n",execname(),pid(),$info->node)  // 结构体成员
}

struct fb_info {
...
    void *par;
...
}
struct psb_fbdev {
    struct drm_fb_helper psb_fb_helper;
    struct psb_framebuffer pfb;
};
#!/usr/bin/stap
probe kernel.statement("fb_mmap@drivers/video/fbmem.c:1357")
{
    p = @cast($info->par, "struct psb_fbdev")->pfb
    printf("%s(%d) %d\n",execname(),pid(),p)
}

try...catch

try { 
   /* do something */
   /* trigger error like kread(0), or divide by zero, or error("foo") */
} catch (msg) { /* omit (msg) entirely if not interested */
   /* println("caught error ", msg) */
   /* handle error */
}
/* execution continues */

foreach

// The array must not be modified within the statement. If you add a single plus (+) or minus (-) operator after the VAR or the ARRAY identifier, the iteration order will be sorted by the ascending or descending index or value
foreach (VAR in ARRAY) STMT

// The following statement behaves the same as the first example, except it is used when an array is indexed with a tuple of keys. Use a sorting suffix on at most one VAR or ARRAY identifier.
foreach ([VAR1, VAR2, ...] in ARRAY) STMT

// You can combine the first and second syntax to capture both the full tuple and the keys at the same time as follows.

foreach (VAR = [VAR1, VAR2, ...] in ARRAY) STMT

foreach (VAR in ARRAY limit EXP) STMT

• limit keyword limits the number of loop iterations to EXP times

  function

  语法：

  function <name>[:<type>] ( <arg1>[:<type>], ... )[:<priority>] { <stmts> }

  基本的例子：

function <name>[:<type>] ( <arg1>[:<type>], ... )[:<priority>] { <stmts> }

function thisfn (arg1, arg2) {
    return arg1 + arg2
}

function thatfn:string(arg1:long, arg2) {
    return sprintf("%d%s", arg1, arg2)
}

private function three:long () { return 3 } // 仅仅是定义的文件才能够使用

// 允许函数重载
function g() { print("first function") }
function g(x) { print("second function") }
g() -> "first function"
g(1) -> "second function"  // 形参不同，在编译期间进行重载

• tid() the id of current thread
• uid() the id of the current user
• cpu() the current cpu number
• ctime() convert number of seconds since UNIX epoch to date
• pp() A string describing the probe point currently being handled.
• thread_indent()

  嵌入C

语法

function <name>:<type> ( <arg1>:<type>, ... )[:<priority>] %{ <C_stmts> %}

所有的在%{和%}之间的，都是C代码。在使用的时候，需要格外小心。

具体的例子：

%{
#include <linux/in.h>
#include <linux/ip.h>
%} /* <-- top level */

/* Reads the char value stored at a given address: */ 
function __read_char:long(addr:long) %{ /* pure */
         STAP_RETURN(kderef(sizeof(char), STAP_ARG_addr));
         CATCH_DEREF_FAULT ();
%} /* <-- function body */

/* Determines whether an IP packet is TCP, based on the iphdr: */
function is_tcp_packet:long(iphdr) {
         protocol = @cast(iphdr, "iphdr")->protocol
         return (protocol == %{ IPPROTO_TCP %}) /* <-- expression */
}

• 嵌入C的时候，返回值类型一定要给出。
• 在嵌入C的语言中，对函数的输入和输出（形参和返回值）的参数用STAP_ARG_foo和STAP_RETVALUE。

function integer_ops:long (val) %{
  STAP_PRINTF("%d\n", STAP_ARG_val);
  STAP_RETVALUE = STAP_ARG_val + 1;
  if (STAP_RETVALUE == 4)
      STAP_ERROR("wrong guess: %d", (int) STAP_RETVALUE);
  if (STAP_RETVALUE == 3)
      STAP_RETURN(0);
  STAP_RETVALUE ++;
%}
function string_ops:string (val) %{
  strlcpy (STAP_RETVALUE, STAP_ARG_val, MAXSTRINGLEN);
  strlcat (STAP_RETVALUE, "one", MAXSTRINGLEN);
  if (strcmp (STAP_RETVALUE, "three-two-one"))
      STAP_RETURN("parameter should be three-two-");
%}
function no_ops () %{
    STAP_RETURN(); /* function inferred with no return value */
%}

嵌入的评论

• /* pure */ 表示嵌入的C语言没有副作用
• /* stable */ means that the C code always has the same value.
• /* guru */ means that the C code is so unsafe that a systemtap user must specify -g，必须使用-g参数

嵌入C语言中访问全局变量

global var
global var2[100]
function increment() %{
    /* pragma:read:var */ /* pragma:write:var */     // 必须显示的使用read
    /* pragma:read:var2 */ /* pragma:write:var2 */    // 必须显示使用的write
    STAP_GLOBAL_SET_var(STAP_GLOBAL_GET_var()+1); //var++
    STAP_GLOBAL_SET_var2(1, 1, STAP_GLOBAL_GET_var2(1, 1)+1); //var2[1,1]++
%}

常见的probes

• probe.nevel handler从不执行
• probe.begin, end, error

probe error,end {
  if (varerr) printf("%s %s access errors: %d", @1, @2, varerr);
}

选项

• -w 不显示所有的warning
• -c 用来执行探测的命令
• -x trace only speci ed PID (only for userland probing)
• -L list probe points matching given pattern along with available variables
• -g embed C code in stap script  unsafe, dangerous and fun

资料来源

• https://segmentfault.com/a/1190000010774974

定位函数位置

root@j9 ~# stap -l 'process("/lib/x86_64-linux-gnu/libc.so.6").function("printf")' 
process("/lib/x86_64-linux-gnu/libc-2.15.so").function("__printf@/build/buildd/eglibc-2.15/stdio-common/printf.c:29")

可以看出printf是在printf.c第29行定义的。再比如要看内核中recv系统的调用是在哪里定义的：

root@j9 ~# stap -l 'kernel.function("sys_recv")'
kernel.function("sys_recv@/build/buildd/linux-lts-trusty-3.13.0/net/socket.c:1868")

同理，也可以用来定位用户进程的函数位置： 比如tengine的文件ngx_shmem.c里面为了兼容各个操作系统而实现了三个版本的ngx_shm_alloc，用#if (NGX_HAVE_MAP_ANON)、#elif (NGX_HAVE_MAP_DEVZERO)、#elif (NGX_HAVE_SYSVSHM)、#endif来做条件编译，那怎么知道编译出来的是哪个版本呢，用SystemTap的话就很简单了，否则要去grep一下这几宏有没有定义才知道了。

[root@cache4 tengine]# stap -l 'process("/home/admin/tengine/bin/nginx").function("ngx_shm_alloc")'
process("/home/admin/tengine/bin/nginx").function("ngx_shm_alloc@src/os/unix/ngx_shmem.c:15")

例子

#! /usr/bin/env stap

global var, varerr

probe $1 {
  // 如果是定义的参数
  if (@defined($2)) {
     try {
         newvar = $2;
         if (var[tid()] != newvar) {
            printf("%s[%d] %s %s:\n", execname(), tid(), pp(), @2);
            println(newvar);
            var[tid()] = newvar;
         }
     } catch { varerr ++ }  # error during $2 resolution or perhaps var[] assignment
  }
}

probe kprocess.release { # if using per-thread checking
  delete var[$p->pid] # thread
}

probe never {
  var[0]=""  # assigns a type to var in case no probes match $1 above
}

probe error,end {
  if (varerr) printf("%s %s access errors: %d", @1, @2, varerr);
}

使用

# -w用来将警告忽略，-c表示监控的程序
stap -w varwatch.stp 'kernel.function("sys_*@fs/open.c:*")' '$$parms' -c "ls > /dev/null"

关联数组

foo["tom"] = 23
foo["dict"] = 24
foo["harry"] = 25
device[pid(),execname(),uid(),ppid(),"W"] = devname // 关联了多个值作为key
global foo # foo gets the default size
global bar[100] # bar will have 100 slots

if (42 in foo) println("foo has 42")
if (["bar", 42] in foo) println("foo has [\"bar\", 42]")
foreach (x in foo) printf("foo[%d] = %d\n", x, foo[x])
foreach ([s, x] in foo)
    printf("foo[%s, %d] = %d\n", s, x, foo[x])

命令行参数

probe kernel.function(@1) { } // @表示字符串，$表示数字
probe kernel.function(@1).return { }

• Use $1 ... $ for pasting the entire argument string into the input stream, which will be further lexically tokenized.
• Use @1 ... @ for casting an entire argument as a string literal.
• Use $# for the number of arguments (or @# as a string)

probe begin { printf("%d, %s\n"), $1, @2) }

使用：

# stap example.stp '5+5' mystring
10, mystring

kernel profiling

global profile, pcount
probe timer.profile {
  pcount <<< 1  // +=1
  fn = user_mode() ? "<user>" : symname(addr())
  if (fn != "") profile[fn] <<< 1  
}
probe timer.ms(4000) {
  printf ("\n--- %d samples recorded:\n", @count(pcount))
  foreach (f in profile- limit 10) {
    printf ("%s\t%d\n", f, @count(profile[f]))
  }
  delete profile
  delete pcount
}

条件编译

condition条件

%( CONDITION %? TRUE-TOKENS %)
%( CONDITION %? TRUE-TOKENS %: FALSE-TOKENS %)

%( <condition> %? <code> [ %: <code> ] %)

@defined

• The predicate @defined() is available for testing whether a particular $variable/expression is resolvable at translation time. The following is an example of its use:

preprocessor macros

语法：

@define NAME %( BODY %)
@define NAME(PARAM_1, PARAM_2, ...) %( BODY %)

引用的时候，在符号的名称上加上@

@define foo %( x %)
@define add(a,b) %( ((@a)+(@b)) %)
@foo = @add(2,2)

• Macro expansion is currently performed in a separate pass before conditional compilation

所以下面的代码是错误的：

// The following results in a conflict:
%( CONFIG_UTRACE == "y" %?
    @define foo %( process.syscall %)
%:
    @define foo %( **ERROR** %)
%)

// The following works properly as expected:
@define foo %(
  %( CONFIG_UTRACE == "y" %? process.syscall %: **ERROR** %)
%)

查看当前文件的探测点

#include <stdio.h>
int main(int argc, char* argv[]) {
    int a = 10;
    ++a;
    int b = 20;
    ++b;
    return 0;
}

stap -L 'process("/home/brookchen/Github/test").function("*")'
process("/home/brookchen/Github/test").function("__do_global_dtors_aux")
process("/home/brookchen/Github/test").function("__libc_csu_fini")
process("/home/brookchen/Github/test").function("__libc_csu_init")
process("/home/brookchen/Github/test").function("_fini")
process("/home/brookchen/Github/test").function("_init")
process("/home/brookchen/Github/test").function("_start")
process("/home/brookchen/Github/test").function("deregister_tm_clones")
process("/home/brookchen/Github/test").function("frame_dummy")
process("/home/brookchen/Github/test").function("main@/home/brookchen/Github/test.c:2") $argc:int $argv:char** $a:int $b:int
process("/home/brookchen/Github/test").function("register_tm_clones")

target_variable

stap -L 'kernel.function("vfs_read")'
kernel.function("vfs_read@./fs/read_write.c:460") $file:struct file* $buf:char* $count:size_t $pos:loff_t* $ret:ssize_t

• 这里每个target variable 都是通过$ + 变量名，冒号后面是类型

访问静态全局变量target_variable

stap -e 'probe kernel.function("vfs_read") {
           printf ("current files_stat max_files: %d\n",
                   @var("files_stat@fs/file_table.c")->max_files);
           exit(); }'

一个简单的例子来练习stap

// tac.c
#include <stdio.h>
#include <stdio.h>
#include <stddef.h>
#include <string.h>

char* haha = "wahaha\n";
char* read_line(FILE* fp, char* buf, size_t len) {
    return fgets(buf, len, fp);
}
char* reverse_line(char* line, size_t l) {
    char *s = line, *e = s + l - sizeof("\n"), t;
    while(s < e) {
        t =*s, *s = *e, *e = t; s++, e--;
    }
    return line;
}
void write_line(char* line){ fputs(line, stdout);}

int main(int argc, char * argv[]) {
        char buf[4096], *line; FILE* fp = stdin;
    if(argc != 1 ) {
                fp = fopen(argv[1], "r");
    }
    if(fp == NULL) {
        fprintf(stdout, "usage: %s filename\n", argv[0]);return -1;}

    while((line = read_line(fp, buf, sizeof(buf)))) {
        line = reverse_line(line, strlen(line));
        write_line(line);
    }

    if(argc != 1) {
        fclose(fp);
    }
    return 0;
}

查看可用的探测点

brookchen@brookchen:~/Github$ sudo stap -L 'process("/home/brookchen/Github/tac").function("*")'
process("/home/brookchen/Github/tac").function("__do_global_dtors_aux")
process("/home/brookchen/Github/tac").function("__libc_csu_fini")
process("/home/brookchen/Github/tac").function("__libc_csu_init")
process("/home/brookchen/Github/tac").function("_fini")
process("/home/brookchen/Github/tac").function("_init")
process("/home/brookchen/Github/tac").function("_start")
process("/home/brookchen/Github/tac").function("deregister_tm_clones")
process("/home/brookchen/Github/tac").function("frame_dummy")
process("/home/brookchen/Github/tac").function("main@/home/brookchen/Github/tac.c:20") $argc:int $argv:char** $buf:char[] $line:char* $fp:FILE*
process("/home/brookchen/Github/tac").function("read_line@/home/brookchen/Github/tac.c:7") $fp:FILE* $buf:char* $len:size_t
process("/home/brookchen/Github/tac").function("register_tm_clones")
process("/home/brookchen/Github/tac").function("reverse_line@/home/brookchen/Github/tac.c:10") $line:char* $l:size_t $s:char* $e:char* $t:char
process("/home/brookchen/Github/tac").function("write_line@/home/brookchen/Github/tac.c:17") $line:char*

打出函数调用栈

probe process(@1).function(@2){
    print_ubacktrace();
    exit();
}
# -d