Re: Getting empty callchain from perf_callchain_kernel()
From: Song Liu
Date: Thu Jun 06 2019 - 12:10:59 EST
> On May 27, 2019, at 4:57 AM, Kairui Song <kasong@xxxxxxxxxx> wrote:
>
> On Sat, May 25, 2019 at 7:23 AM Josh Poimboeuf <jpoimboe@xxxxxxxxxx> wrote:
>>
>> On Fri, May 24, 2019 at 10:20:52AM +0800, Kairui Song wrote:
>>> On Fri, May 24, 2019 at 1:27 AM Josh Poimboeuf <jpoimboe@xxxxxxxxxx> wrote:
>>>>
>>>> On Fri, May 24, 2019 at 12:41:59AM +0800, Kairui Song wrote:
>>>>> On Thu, May 23, 2019 at 11:24 PM Josh Poimboeuf <jpoimboe@xxxxxxxxxx> wrote:
>>>>>>
>>>>>> On Thu, May 23, 2019 at 10:50:24PM +0800, Kairui Song wrote:
>>>>>>>>> Hi Josh, this still won't fix the problem.
>>>>>>>>>
>>>>>>>>> Problem is not (or not only) with ___bpf_prog_run, what actually went
>>>>>>>>> wrong is with the JITed bpf code.
>>>>>>>>
>>>>>>>> There seem to be a bunch of issues. My patch at least fixes the failing
>>>>>>>> selftest reported by Alexei for ORC.
>>>>>>>>
>>>>>>>> How can I recreate your issue?
>>>>>>>
>>>>>>> Hmm, I used bcc's example to attach bpf to trace point, and with that
>>>>>>> fix stack trace is still invalid.
>>>>>>>
>>>>>>> CMD I used with bcc:
>>>>>>> python3 ./tools/stackcount.py t:sched:sched_fork
>>>>>>
>>>>>> I've had problems in the past getting bcc to build, so I was hoping it
>>>>>> was reproducible with a standalone selftest.
>>>>>>
>>>>>>> And I just had another try applying your patch, self test is also failing.
>>>>>>
>>>>>> Is it the same selftest reported by Alexei?
>>>>>>
>>>>>> test_stacktrace_map:FAIL:compare_map_keys stackid_hmap vs. stackmap err -1 errno 2
>>>>>>
>>>>>>> I'm applying on my local master branch, a few days older than
>>>>>>> upstream, I can update and try again, am I missing anything?
>>>>>>
>>>>>> The above patch had some issues, so with some configs you might see an
>>>>>> objtool warning for ___bpf_prog_run(), in which case the patch doesn't
>>>>>> fix the test_stacktrace_map selftest.
>>>>>>
>>>>>> Here's the latest version which should fix it in all cases (based on
>>>>>> tip/master):
>>>>>>
>>>>>> https://git.kernel.org/pub/scm/linux/kernel/git/jpoimboe/linux.git/commit/?h=bpf-orc-fix
>>>>>
>>>>> Hmm, I still get the failure:
>>>>> test_stacktrace_map:FAIL:compare_map_keys stackid_hmap vs. stackmap
>>>>> err -1 errno 2
>>>>>
>>>>> And I didn't see how this will fix the issue. As long as ORC need to
>>>>> unwind through the JITed code it will fail. And that will happen
>>>>> before reaching ___bpf_prog_run.
>>>>
>>>> Ok, I was able to recreate by doing
>>>>
>>>> echo 1 > /proc/sys/net/core/bpf_jit_enable
>>>>
>>>> first. I'm guessing you have CONFIG_BPF_JIT_ALWAYS_ON.
>>>>
>>>
>>> Yes, with JIT off it will be fixed. I can confirm that.
>>
>> Here's a tentative BPF fix for the JIT frame pointer issue. It was a
>> bit harder than I expected. Encoding r12 as a base register requires a
>> SIB byte, so I had to add support for encoding that. I also simplified
>> the prologue to resemble a GCC prologue, which decreases the prologue
>> size quite a bit.
>>
>> Next week I can work on the corresponding ORC change. Then I can clean
>> all the patches up and submit them properly.
Hi Josh,
Have you got luck fixing the ORC side?
Thanks,
Song
>>
>> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
>> index afabf597c855..c9b4503558c9 100644
>> --- a/arch/x86/net/bpf_jit_comp.c
>> +++ b/arch/x86/net/bpf_jit_comp.c
>> @@ -104,9 +104,8 @@ static int bpf_size_to_x86_bytes(int bpf_size)
>> /*
>> * The following table maps BPF registers to x86-64 registers.
>> *
>> - * x86-64 register R12 is unused, since if used as base address
>> - * register in load/store instructions, it always needs an
>> - * extra byte of encoding and is callee saved.
>> + * RBP isn't used; it needs to be preserved to allow the unwinder to move
>> + * through generated code stacks.
>> *
>> * Also x86-64 register R9 is unused. x86-64 register R10 is
>> * used for blinding (if enabled).
>> @@ -122,7 +121,7 @@ static const int reg2hex[] = {
>> [BPF_REG_7] = 5, /* R13 callee saved */
>> [BPF_REG_8] = 6, /* R14 callee saved */
>> [BPF_REG_9] = 7, /* R15 callee saved */
>> - [BPF_REG_FP] = 5, /* RBP readonly */
>> + [BPF_REG_FP] = 4, /* R12 readonly */
>> [BPF_REG_AX] = 2, /* R10 temp register */
>> [AUX_REG] = 3, /* R11 temp register */
>> };
>> @@ -139,6 +138,7 @@ static bool is_ereg(u32 reg)
>> BIT(BPF_REG_7) |
>> BIT(BPF_REG_8) |
>> BIT(BPF_REG_9) |
>> + BIT(BPF_REG_FP) |
>> BIT(BPF_REG_AX));
>> }
>>
>> @@ -147,6 +147,11 @@ static bool is_axreg(u32 reg)
>> return reg == BPF_REG_0;
>> }
>>
>> +static bool is_sib_reg(u32 reg)
>> +{
>> + return reg == BPF_REG_FP;
>> +}
>> +
>> /* Add modifiers if 'reg' maps to x86-64 registers R8..R15 */
>> static u8 add_1mod(u8 byte, u32 reg)
>> {
>> @@ -190,15 +195,13 @@ struct jit_context {
>> #define BPF_MAX_INSN_SIZE 128
>> #define BPF_INSN_SAFETY 64
>>
>> -#define AUX_STACK_SPACE 40 /* Space for RBX, R13, R14, R15, tailcnt */
>> -
>> -#define PROLOGUE_SIZE 37
>> +#define PROLOGUE_SIZE 25
>>
>> /*
>> * Emit x86-64 prologue code for BPF program and check its size.
>> * bpf_tail_call helper will skip it while jumping into another program
>> */
>> -static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
>> +static void emit_prologue(u8 **pprog, u32 stack_depth)
>> {
>> u8 *prog = *pprog;
>> int cnt = 0;
>> @@ -206,40 +209,67 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf)
>> /* push rbp */
>> EMIT1(0x55);
>>
>> - /* mov rbp,rsp */
>> + /* mov rbp, rsp */
>> EMIT3(0x48, 0x89, 0xE5);
>>
>> - /* sub rsp, rounded_stack_depth + AUX_STACK_SPACE */
>> - EMIT3_off32(0x48, 0x81, 0xEC,
>> - round_up(stack_depth, 8) + AUX_STACK_SPACE);
>> + /* push r15 */
>> + EMIT2(0x41, 0x57);
>> + /* push r14 */
>> + EMIT2(0x41, 0x56);
>> + /* push r13 */
>> + EMIT2(0x41, 0x55);
>> + /* push r12 */
>> + EMIT2(0x41, 0x54);
>> + /* push rbx */
>> + EMIT1(0x53);
>>
>> - /* sub rbp, AUX_STACK_SPACE */
>> - EMIT4(0x48, 0x83, 0xED, AUX_STACK_SPACE);
>> + /*
>> + * Push the tail call counter (tail_call_cnt) for eBPF tail calls.
>> + * Initialized to zero.
>> + *
>> + * push $0
>> + */
>> + EMIT2(0x6a, 0x00);
>>
>> - /* mov qword ptr [rbp+0],rbx */
>> - EMIT4(0x48, 0x89, 0x5D, 0);
>> - /* mov qword ptr [rbp+8],r13 */
>> - EMIT4(0x4C, 0x89, 0x6D, 8);
>> - /* mov qword ptr [rbp+16],r14 */
>> - EMIT4(0x4C, 0x89, 0x75, 16);
>> - /* mov qword ptr [rbp+24],r15 */
>> - EMIT4(0x4C, 0x89, 0x7D, 24);
>> + /*
>> + * R12 is used for the BPF program's FP register. It points to the end
>> + * of the program's stack area.
>> + *
>> + * mov r12, rsp
>> + */
>> + EMIT3(0x49, 0x89, 0xE4);
>>
>> - if (!ebpf_from_cbpf) {
>> - /*
>> - * Clear the tail call counter (tail_call_cnt): for eBPF tail
>> - * calls we need to reset the counter to 0. It's done in two
>> - * instructions, resetting RAX register to 0, and moving it
>> - * to the counter location.
>> - */
>> + /* sub rsp, rounded_stack_depth */
>> + EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8));
>>
>> - /* xor eax, eax */
>> - EMIT2(0x31, 0xc0);
>> - /* mov qword ptr [rbp+32], rax */
>> - EMIT4(0x48, 0x89, 0x45, 32);
>> + BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
>>
>> - BUILD_BUG_ON(cnt != PROLOGUE_SIZE);
>> - }
>> + *pprog = prog;
>> +}
>> +
>> +static void emit_epilogue(u8 **pprog)
>> +{
>> + u8 *prog = *pprog;
>> + int cnt = 0;
>> +
>> + /* lea rsp, [rbp-0x28] */
>> + EMIT4(0x48, 0x8D, 0x65, 0xD8);
>> +
>> + /* pop rbx */
>> + EMIT1(0x5B);
>> + /* pop r12 */
>> + EMIT2(0x41, 0x5C);
>> + /* pop r13 */
>> + EMIT2(0x41, 0x5D);
>> + /* pop r14 */
>> + EMIT2(0x41, 0x5E);
>> + /* pop r15 */
>> + EMIT2(0x41, 0x5F);
>> + /* pop rbp */
>> + EMIT1(0x5D);
>> +
>> + /* ret */
>> + EMIT1(0xC3);
>>
>> *pprog = prog;
>> }
>> @@ -277,7 +307,7 @@ static void emit_bpf_tail_call(u8 **pprog)
>> EMIT2(0x89, 0xD2); /* mov edx, edx */
>> EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */
>> offsetof(struct bpf_array, map.max_entries));
>> -#define OFFSET1 (41 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */
>> +#define OFFSET1 (35 + RETPOLINE_RAX_BPF_JIT_SIZE) /* Number of bytes to jump */
>> EMIT2(X86_JBE, OFFSET1); /* jbe out */
>> label1 = cnt;
>>
>> @@ -285,13 +315,13 @@ static void emit_bpf_tail_call(u8 **pprog)
>> * if (tail_call_cnt > MAX_TAIL_CALL_CNT)
>> * goto out;
>> */
>> - EMIT2_off32(0x8B, 0x85, 36); /* mov eax, dword ptr [rbp + 36] */
>> + EMIT3(0x8B, 0x45, 0xD4); /* mov eax, dword ptr [rbp - 44] */
>> EMIT3(0x83, 0xF8, MAX_TAIL_CALL_CNT); /* cmp eax, MAX_TAIL_CALL_CNT */
>> -#define OFFSET2 (30 + RETPOLINE_RAX_BPF_JIT_SIZE)
>> +#define OFFSET2 (27 + RETPOLINE_RAX_BPF_JIT_SIZE)
>> EMIT2(X86_JA, OFFSET2); /* ja out */
>> label2 = cnt;
>> EMIT3(0x83, 0xC0, 0x01); /* add eax, 1 */
>> - EMIT2_off32(0x89, 0x85, 36); /* mov dword ptr [rbp + 36], eax */
>> + EMIT3(0x89, 0x45, 0xD4); /* mov dword ptr [rbp - 44], eax */
>>
>> /* prog = array->ptrs[index]; */
>> EMIT4_off32(0x48, 0x8B, 0x84, 0xD6, /* mov rax, [rsi + rdx * 8 + offsetof(...)] */
>> @@ -419,8 +449,7 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
>> int proglen = 0;
>> u8 *prog = temp;
>>
>> - emit_prologue(&prog, bpf_prog->aux->stack_depth,
>> - bpf_prog_was_classic(bpf_prog));
>> + emit_prologue(&prog, bpf_prog->aux->stack_depth);
>>
>> for (i = 0; i < insn_cnt; i++, insn++) {
>> const s32 imm32 = insn->imm;
>> @@ -767,10 +796,19 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
>> case BPF_ST | BPF_MEM | BPF_DW:
>> EMIT2(add_1mod(0x48, dst_reg), 0xC7);
>>
>> -st: if (is_imm8(insn->off))
>> - EMIT2(add_1reg(0x40, dst_reg), insn->off);
>> +st:
>> + if (is_imm8(insn->off))
>> + EMIT1(add_1reg(0x40, dst_reg));
>> else
>> - EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);
>> + EMIT1(add_1reg(0x80, dst_reg));
>> +
>> + if (is_sib_reg(dst_reg))
>> + EMIT1(add_1reg(0x20, dst_reg));
>> +
>> + if (is_imm8(insn->off))
>> + EMIT1(insn->off);
>> + else
>> + EMIT(insn->off, 4);
>>
>> EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
>> break;
>> @@ -799,11 +837,19 @@ st: if (is_imm8(insn->off))
>> goto stx;
>> case BPF_STX | BPF_MEM | BPF_DW:
>> EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);
>> -stx: if (is_imm8(insn->off))
>> - EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
>> +stx:
>> + if (is_imm8(insn->off))
>> + EMIT1(add_2reg(0x40, dst_reg, src_reg));
>> + else
>> + EMIT1(add_2reg(0x80, dst_reg, src_reg));
>> +
>> + if (is_sib_reg(dst_reg))
>> + EMIT1(add_1reg(0x20, dst_reg));
>> +
>> + if (is_imm8(insn->off))
>> + EMIT1(insn->off);
>> else
>> - EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
>> - insn->off);
>> + EMIT(insn->off, 4);
>> break;
>>
>> /* LDX: dst_reg = *(u8*)(src_reg + off) */
>> @@ -825,16 +871,24 @@ stx: if (is_imm8(insn->off))
>> case BPF_LDX | BPF_MEM | BPF_DW:
>> /* Emit 'mov rax, qword ptr [rax+0x14]' */
>> EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
>> -ldx: /*
>> +ldx:
>> + /*
>> * If insn->off == 0 we can save one extra byte, but
>> * special case of x86 R13 which always needs an offset
>> * is not worth the hassle
>> */
>> if (is_imm8(insn->off))
>> - EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off);
>> + EMIT1(add_2reg(0x40, src_reg, dst_reg));
>> else
>> - EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),
>> - insn->off);
>> + EMIT1(add_2reg(0x80, src_reg, dst_reg));
>> +
>> + if (is_sib_reg(src_reg))
>> + EMIT1(add_1reg(0x20, src_reg));
>> +
>> + if (is_imm8(insn->off))
>> + EMIT1(insn->off);
>> + else
>> + EMIT(insn->off, 4);
>> break;
>>
>> /* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */
>> @@ -847,11 +901,19 @@ stx: if (is_imm8(insn->off))
>> goto xadd;
>> case BPF_STX | BPF_XADD | BPF_DW:
>> EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01);
>> -xadd: if (is_imm8(insn->off))
>> - EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
>> +xadd:
>> + if (is_imm8(insn->off))
>> + EMIT1(add_2reg(0x40, dst_reg, src_reg));
>> else
>> - EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
>> - insn->off);
>> + EMIT1(add_2reg(0x80, dst_reg, src_reg));
>> +
>> + if (is_sib_reg(dst_reg))
>> + EMIT1(add_1reg(0x20, dst_reg));
>> +
>> + if (is_imm8(insn->off))
>> + EMIT1(insn->off);
>> + else
>> + EMIT(insn->off, 4);
>> break;
>>
>> /* call */
>> @@ -1040,19 +1102,8 @@ xadd: if (is_imm8(insn->off))
>> seen_exit = true;
>> /* Update cleanup_addr */
>> ctx->cleanup_addr = proglen;
>> - /* mov rbx, qword ptr [rbp+0] */
>> - EMIT4(0x48, 0x8B, 0x5D, 0);
>> - /* mov r13, qword ptr [rbp+8] */
>> - EMIT4(0x4C, 0x8B, 0x6D, 8);
>> - /* mov r14, qword ptr [rbp+16] */
>> - EMIT4(0x4C, 0x8B, 0x75, 16);
>> - /* mov r15, qword ptr [rbp+24] */
>> - EMIT4(0x4C, 0x8B, 0x7D, 24);
>> -
>> - /* add rbp, AUX_STACK_SPACE */
>> - EMIT4(0x48, 0x83, 0xC5, AUX_STACK_SPACE);
>> - EMIT1(0xC9); /* leave */
>> - EMIT1(0xC3); /* ret */
>> +
>> + emit_epilogue(&prog);
>> break;
>>
>> default:
>
> Thanks! This looks good to me and passed the self test and bcc test
> (with frame pointer unwinder, and JIT enabled):
> With bcc's tools/stackcount.py I got the valid stack trace, and the
> self test says:
> test_stacktrace_map:PASS:compare_map_keys stackid_hmap vs. stackmap 0 nsec
>
> --
> Best Regards,
> Kairui Song