使用 bpf 跟踪 bpf map 的更新、删除函数，以此确认是谁动了 bpf map。
TL;DR 已实现对 bpf map 的更新、删除函数的跟踪，未实现批量操作 bpf map 函数的跟踪；源代码：github.com/Asphaltt/mad。
实现效果
跟踪 pwru 的配置：
# ./mad
2024/11/17 07:06:47 mad is running ..
0: map(5755:.rodata:Array) is updated by process(76052:pwru)
key: 00000000 [4 bytes]
0: map(5755:.rodata:Array) is updated by process(76052:pwru)
value: {".rodata": [{"CFG": {"netns": 0, "mark": 0, "ifindex": 0, "output_meta": 0x1, "output_tuple": 0x1, "output_skb": 0x0, "output_shinfo": 0x1, "output_stack": 0x0, "output_caller": 0x0, "output_unused": 0x0, "is_set": 0x1, "track_skb": 0x0, "track_skb_by_stackid": 0x0, "track_xdp": 0x1, "unused": 0x0, "skb_btf_id": 1875, "shinfo_btf_id": 21388}}, {"TRUE": true}, {"ZERO": 0}, {"BPF_PROG_ADDR": 0}]}
即使是字符串，也能识别出来：
3: map(5766:retsnoop.data:Array) is updated by process(76100:retsnoop)
key: 00000000 [4 bytes]
3: map(5766:retsnoop.data:Array) is updated by process(76100:retsnoop)
value: {".data": [{"FMT_SUCC_VOID": " EXIT %s%s [VOID] "}, {"FMT_SUCC_TRUE": " EXIT %s%s [true] "}, {"FMT_SUCC_FALSE": " EXIT %s%s [false] "}, {"FMT_FAIL_NULL": "[!] EXIT %s%s [NULL] "}, {"FMT_FAIL_PTR": "[!] EXIT %s%s [%d] "}, {"FMT_SUCC_PTR": " EXIT %s%s [0x%lx] "}, {"FMT_FAIL_LONG": "[!] EXIT %s%s [%ld] "}, {"FMT_SUCC_LONG": " EXIT %s%s [%ld] "}, {"FMT_FAIL_INT": "[!] EXIT %s%s [%d] "}, {"FMT_SUCC_INT": " EXIT %s%s [%d] "}, {"FMT_SUCC_VOID_COMPAT": " EXIT [%d] %s [VOID] "}, {"FMT_SUCC_TRUE_COMPAT": " EXIT [%d] %s [true] "}, {"FMT_SUCC_FALSE_COMPAT": " EXIT [%d] %s [false] "}, {"FMT_FAIL_NULL_COMPAT": "[!] EXIT [%d] %s [NULL] "}, {"FMT_FAIL_PTR_COMPAT": "[!] EXIT [%d] %s [%d] "}, {"FMT_SUCC_PTR_COMPAT": " EXIT [%d] %s [0x%lx] "}, {"FMT_FAIL_LONG_COMPAT": "[!] EXIT [%d] %s [%ld] "}, {"FMT_SUCC_LONG_COMPAT": " EXIT [%d] %s [%ld] "}, {"FMT_FAIL_INT_COMPAT": "[!] EXIT [%d] %s [%d] "}, {"FMT_SUCC_INT_COMPAT": " EXIT [%d] %s [%d] "}, {"push_call_stack.___fmt": "=== STARTING TRACING %s [COMM %s PID %d] === "}, {"push_call_stack.___fmt.1": " ENTER %s%s [...] "}, {"push_call_stack.___fmt.2": "=== STARTING TRACING %s [PID %d] === "}, {"push_call_stack.___fmt.3": "=== ... TRACING [PID %d COMM %s] === "}, {"push_call_stack.___fmt.4": " ENTER [%d] %s [...] "}, {"save_stitch_stack.___fmt": "SHOULDN'T HAPPEN DEPTH %ld LEN %ld
"}, {"save_stitch_stack.___fmt.5": "CURRENT DEPTH %d..%d "}, {"save_stitch_stack.___fmt.6": "SAVED DEPTH %d..%d "}, {"save_stitch_stack.___fmt.7": "STITCHED STACK %d..%d to ..%d
"}, {"save_stitch_stack.___fmt.8": "EMIT PARTIAL STACK DEPTH %d..%d
"}, {"save_stitch_stack.___fmt.9": "RESETTING SAVED ERR STACK %d..%d to %d..
"}, {"pop_call_stack.___fmt": "POP(0) UNEXPECTED PID %d DEPTH %d MAX DEPTH %d "}, {"pop_call_stack.___fmt.10": "POP(1) UNEXPECTED GOT ID %d ADDR %lx NAME %s "}, {"pop_call_stack.___fmt.11": "POP(2) UNEXPECTED WANT ID %u ADDR %lx NAME %s "}, {"pop_call_stack.___fmt.12": "EMIT ERROR STACK DEPTH %d (SAVED ..%d)
"}, {"pop_call_stack.___fmt.13": "EMIT SUCCESS STACK DEPTH %d (SAVED ..%d)
"}]}
实现细节之识别目标函数
在 mad 里，并没有写死所有要跟踪的内核函数，而是通过 BTF 去识别这些函数：
// https://github.com/Asphaltt/mad/blob/main/bpf_btf.go
func isTgtFunc(typ btf.Type) (string, bool) {
fn, ok := typ.(*btf.Func)
if !ok {
return "", false
}
fnName := fn.Name
fnProto := fn.Type.(*btf.FuncProto)
if strings.HasSuffix(fnName, "_map_update_elem") && len(fnProto.Params) == 4 {
if isBpfMap(fnProto.Params[0].Type) && isVoid(fnProto.Params[1].Type) && isVoid(fnProto.Params[2].Type) {
return fnName, true
}
} else if strings.HasSuffix(fnName, "_map_delete_elem") && len(fnProto.Params) == 2 {
if isBpfMap(fnProto.Params[0].Type) && isVoid(fnProto.Params[1].Type) {
return fnName, true
}
}
return "", false
}
func retrieveHooks(spec *btf.Spec) ([]string, error) {
var hooks []string
iter := spec.Iterate()
for iter.Next() {
if name, ok := isTgtFunc(iter.Type); ok {
hooks = append(hooks, name)
}
}
return hooks, nil
}
如上代码片段的处理逻辑：
遍历整个 BTF spec。 判断遍历的 BTF 类型是不是函数。 判断是不是更新 bpf map 的函数：函数名以 _map_update_elem结尾且函数参数符合预期。判断是不是删除 bpf map 的函数：函数名以 _map_delete_elem结尾且函数参数符合预期。
实现细节之根据 BTF 信息解析 kv 数据
使用 bpftool 命令查看 BPF map 的 key 和 value 的数据结构：
# bpftool m d i 5672
[{
"value": {
".rodata": [{
"CFG": {
"netns": 0,
"mark": 0,
"ifindex": 0,
"output_meta": 0x1,
"output_tuple": 0x1,
"output_skb": 0x0,
"output_shinfo": 0x1,
"output_stack": 0x0,
"output_caller": 0x0,
"output_unused": 0x0,
"is_set": 0x1,
"track_skb": 0x0,
"track_skb_by_stackid": 0x0,
"track_xdp": 0x1,
"unused": 0x0,
"skb_btf_id": 1875,
"shinfo_btf_id": 21388
}
},{
"TRUE": true
},{
"ZERO": 0
},{
"BPF_PROG_ADDR": 0
}
]
}
}
]
在 mad 里，能否实现同样的效果呢？
是个苦力活，基本上抄一下 bpftool 的实现就好：
// https://github.com/Asphaltt/mybtf/blob/main/dump.go
func (dd *dataDumper) dumpData(typ btf.Type, bits bitsInfo, data []byte) error {
switch v := typ.(type) {
case *btf.Int:
dd.dumpInt(v, bits, data)
case *btf.Struct:
dd.dumpStruct(v, data)
case *btf.Union:
dd.dumpUnion(v, data)
case *btf.Array:
dd.dumpArray(v, data)
case *btf.Enum:
dd.dumpEnum(v, data)
case *btf.Pointer:
dd.dumpPointer(v, data)
case *btf.Fwd:
dd.print("(fwd-kind-invalid)")
return fmt.Errorf("fwd kind invalid")
case *btf.Typedef:
return dd.dumpData(v.Type, bits, data)
case *btf.Volatile:
return dd.dumpData(v.Type, bits, data)
case *btf.Const:
return dd.dumpData(v.Type, bits, data)
case *btf.Restrict:
return dd.dumpData(v.Type, bits, data)
case *btf.Var:
dd.dumpVar(v, bits, data)
case *btf.Datasec:
return dd.dumpDataSec(v, data)
default:
dd.print("(unsupported-kind)")
return fmt.Errorf("unsupported kind %T", v)
}
return nil
}
以上，便实现了根据 BTF 信息解析数据的功能。
总结
通过 BTF 信息，可以清楚地看出更新、删除 bpf map 的 kv 细节，这对于调试和排查问题是非常有帮助的。
