// SPDX-License-Identifier: GPL-2.0-only /* * eBPF JIT compiler * * Copyright 2016 Naveen N. Rao * IBM Corporation * * Based on the powerpc classic BPF JIT compiler by Matt Evans */ #include #include #include #include #include #include #include #include #include "bpf_jit.h" static void bpf_jit_fill_ill_insns(void *area, unsigned int size) { memset32(area, BREAKPOINT_INSTRUCTION, size / 4); } /* Fix the branch target addresses for subprog calls */ static int bpf_jit_fixup_subprog_calls(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx, u32 *addrs) { const struct bpf_insn *insn = fp->insnsi; bool func_addr_fixed; u64 func_addr; u32 tmp_idx; int i, ret; for (i = 0; i < fp->len; i++) { /* * During the extra pass, only the branch target addresses for * the subprog calls need to be fixed. All other instructions * can left untouched. * * The JITed image length does not change because we already * ensure that the JITed instruction sequence for these calls * are of fixed length by padding them with NOPs. */ if (insn[i].code == (BPF_JMP | BPF_CALL) && insn[i].src_reg == BPF_PSEUDO_CALL) { ret = bpf_jit_get_func_addr(fp, &insn[i], true, &func_addr, &func_addr_fixed); if (ret < 0) return ret; /* * Save ctx->idx as this would currently point to the * end of the JITed image and set it to the offset of * the instruction sequence corresponding to the * subprog call temporarily. */ tmp_idx = ctx->idx; ctx->idx = addrs[i] / 4; bpf_jit_emit_func_call_rel(image, ctx, func_addr); /* * Restore ctx->idx here. This is safe as the length * of the JITed sequence remains unchanged. */ ctx->idx = tmp_idx; } } return 0; } struct powerpc64_jit_data { struct bpf_binary_header *header; u32 *addrs; u8 *image; u32 proglen; struct codegen_context ctx; }; bool bpf_jit_needs_zext(void) { return true; } struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) { u32 proglen; u32 alloclen; u8 *image = NULL; u32 *code_base; u32 *addrs; struct powerpc64_jit_data *jit_data; struct codegen_context cgctx; int pass; int flen; struct bpf_binary_header *bpf_hdr; struct bpf_prog *org_fp = fp; struct bpf_prog *tmp_fp; bool bpf_blinded = false; bool extra_pass = false; if (!fp->jit_requested) return org_fp; tmp_fp = bpf_jit_blind_constants(org_fp); if (IS_ERR(tmp_fp)) return org_fp; if (tmp_fp != org_fp) { bpf_blinded = true; fp = tmp_fp; } jit_data = fp->aux->jit_data; if (!jit_data) { jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); if (!jit_data) { fp = org_fp; goto out; } fp->aux->jit_data = jit_data; } flen = fp->len; addrs = jit_data->addrs; if (addrs) { cgctx = jit_data->ctx; image = jit_data->image; bpf_hdr = jit_data->header; proglen = jit_data->proglen; alloclen = proglen + FUNCTION_DESCR_SIZE; extra_pass = true; goto skip_init_ctx; } addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); if (addrs == NULL) { fp = org_fp; goto out_addrs; } memset(&cgctx, 0, sizeof(struct codegen_context)); memcpy(cgctx.b2p, b2p, sizeof(cgctx.b2p)); /* Make sure that the stack is quadword aligned. */ cgctx.stack_size = round_up(fp->aux->stack_depth, 16); /* Scouting faux-generate pass 0 */ if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { /* We hit something illegal or unsupported. */ fp = org_fp; goto out_addrs; } /* * If we have seen a tail call, we need a second pass. * This is because bpf_jit_emit_common_epilogue() is called * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. */ if (cgctx.seen & SEEN_TAILCALL) { cgctx.idx = 0; if (bpf_jit_build_body(fp, 0, &cgctx, addrs, false)) { fp = org_fp; goto out_addrs; } } bpf_jit_realloc_regs(&cgctx); /* * Pretend to build prologue, given the features we've seen. This will * update ctgtx.idx as it pretends to output instructions, then we can * calculate total size from idx. */ bpf_jit_build_prologue(0, &cgctx); bpf_jit_build_epilogue(0, &cgctx); proglen = cgctx.idx * 4; alloclen = proglen + FUNCTION_DESCR_SIZE; bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns); if (!bpf_hdr) { fp = org_fp; goto out_addrs; } skip_init_ctx: code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); if (extra_pass) { /* * Do not touch the prologue and epilogue as they will remain * unchanged. Only fix the branch target address for subprog * calls in the body. * * This does not change the offsets and lengths of the subprog * call instruction sequences and hence, the size of the JITed * image as well. */ bpf_jit_fixup_subprog_calls(fp, code_base, &cgctx, addrs); /* There is no need to perform the usual passes. */ goto skip_codegen_passes; } /* Code generation passes 1-2 */ for (pass = 1; pass < 3; pass++) { /* Now build the prologue, body code & epilogue for real. */ cgctx.idx = 0; bpf_jit_build_prologue(code_base, &cgctx); if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, extra_pass)) { bpf_jit_binary_free(bpf_hdr); fp = org_fp; goto out_addrs; } bpf_jit_build_epilogue(code_base, &cgctx); if (bpf_jit_enable > 1) pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, proglen - (cgctx.idx * 4), cgctx.seen); } skip_codegen_passes: if (bpf_jit_enable > 1) /* * Note that we output the base address of the code_base * rather than image, since opcodes are in code_base. */ bpf_jit_dump(flen, proglen, pass, code_base); #ifdef PPC64_ELF_ABI_v1 /* Function descriptor nastiness: Address + TOC */ ((u64 *)image)[0] = (u64)code_base; ((u64 *)image)[1] = local_paca->kernel_toc; #endif fp->bpf_func = (void *)image; fp->jited = 1; fp->jited_len = alloclen; bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + (bpf_hdr->pages * PAGE_SIZE)); if (!fp->is_func || extra_pass) { bpf_jit_binary_lock_ro(bpf_hdr); bpf_prog_fill_jited_linfo(fp, addrs); out_addrs: kfree(addrs); kfree(jit_data); fp->aux->jit_data = NULL; } else { jit_data->addrs = addrs; jit_data->ctx = cgctx; jit_data->proglen = proglen; jit_data->image = image; jit_data->header = bpf_hdr; } out: if (bpf_blinded) bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); return fp; }