diff --git a/arch/x86/include/asm/asm-prototypes.h b/arch/x86/include/asm/asm-prototypes.h index 11c6fecc3ad768..255ad64b699f0f 100644 --- a/arch/x86/include/asm/asm-prototypes.h +++ b/arch/x86/include/asm/asm-prototypes.h @@ -6,6 +6,7 @@ #include #include #include +#include #include diff --git a/arch/x86/include/asm/rex.h b/arch/x86/include/asm/rex.h new file mode 100644 index 00000000000000..68d1505079abe3 --- /dev/null +++ b/arch/x86/include/asm/rex.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_REX_UNWIND_H +#define _ASM_X86_REX_UNWIND_H + +#include +#include + +#ifndef __ASSEMBLY__ + +/* + * Total stack is 8 pages (32k) large. 4 pages are reserved for kernel helpers/ + * Therefore, the actual usable stack is 4 pages. + */ +#define REX_STACK_ORDER 3 +#define REX_STACK_SIZE (PAGE_SIZE << REX_STACK_ORDER) + +struct bpf_prog; +struct bpf_insn; + +DECLARE_PER_CPU(unsigned char, rex_termination_state); +DECLARE_PER_CPU(void *, rex_stack_ptr); + +extern asmlinkage unsigned int rex_dispatcher_func( + const void *ctx, + const struct bpf_prog *prog, + unsigned int (*bpf_func)(const void *, + const struct bpf_insn *)); + +int arch_init_rex_stack(void); + +static __always_inline bool arch_on_rex_stack(struct pt_regs *regs) +{ + unsigned long sp = regs->sp; + u64 rex_tos = (u64)this_cpu_read_stable(rex_stack_ptr); + return sp >= (rex_tos + 8 - REX_STACK_SIZE) && sp < (rex_tos + 8); +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* _ASM_X86_REX_UNWIND_H */ diff --git a/arch/x86/include/asm/stacktrace.h b/arch/x86/include/asm/stacktrace.h index 3881b5333eb815..d269b3d4bc62a8 100644 --- a/arch/x86/include/asm/stacktrace.h +++ b/arch/x86/include/asm/stacktrace.h @@ -19,6 +19,7 @@ enum stack_type { STACK_TYPE_IRQ, STACK_TYPE_SOFTIRQ, STACK_TYPE_ENTRY, + STACK_TYPE_REX, STACK_TYPE_EXCEPTION, STACK_TYPE_EXCEPTION_LAST = STACK_TYPE_EXCEPTION + N_EXCEPTION_STACKS-1, }; diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 6c5defd6569a3e..f1590d43df305f 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -17,6 +17,7 @@ #include #include +#include #include static const char * const exception_stack_names[] = { @@ -50,6 +51,9 @@ const char *stack_type_name(enum stack_type type) return "ENTRY_TRAMPOLINE"; } + if (type == STACK_TYPE_REX) + return "REX"; + if (type >= STACK_TYPE_EXCEPTION && type <= STACK_TYPE_EXCEPTION_LAST) return exception_stack_names[type - STACK_TYPE_EXCEPTION]; @@ -167,6 +171,41 @@ static __always_inline bool in_irq_stack(unsigned long *stack, struct stack_info return true; } +static __always_inline bool in_rex_stack(unsigned long *stack, struct stack_info *info) +{ + unsigned long *end = (unsigned long *)this_cpu_read(rex_stack_ptr); + unsigned long *begin; + + /* + * @end points directly to the top most stack entry to avoid a -8 + * adjustment in the stack switch hotpath. Adjust it back before + * calculating @begin. + */ + end++; + begin = end - (IRQ_STACK_SIZE / sizeof(long)); + + /* + * Due to the switching logic RSP can never be == @end because the + * final operation is 'popq %rsp' which means after that RSP points + * to the original stack and not to @end. + */ + if (stack < begin || stack >= end) + return false; + + info->type = STACK_TYPE_REX; + info->begin = begin; + info->end = end; + + /* + * The next stack pointer is stored at the top of the irq stack + * before switching to the irq stack. Actual stack entries are all + * below that. + */ + info->next_sp = (unsigned long *)*(end - 1); + + return true; +} + bool noinstr get_stack_info_noinstr(unsigned long *stack, struct task_struct *task, struct stack_info *info) { @@ -185,6 +224,9 @@ bool noinstr get_stack_info_noinstr(unsigned long *stack, struct task_struct *ta if (in_entry_stack(stack, info)) return true; + if (in_rex_stack(stack, info)) + return true; + return false; } diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index b014e6d229f951..3b2bb3c3a83162 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -259,7 +259,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function) apic_eoi(); trace_call_function_entry(CALL_FUNCTION_VECTOR); inc_irq_stat(irq_call_count); - generic_smp_call_function_interrupt(); + generic_smp_call_function_interrupt(regs); trace_call_function_exit(CALL_FUNCTION_VECTOR); } @@ -268,7 +268,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single) apic_eoi(); trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR); inc_irq_stat(irq_call_count); - generic_smp_call_function_single_interrupt(); + generic_smp_call_function_single_interrupt(regs); trace_call_function_single_exit(CALL_FUNCTION_SINGLE_VECTOR); } diff --git a/arch/x86/net/Makefile b/arch/x86/net/Makefile index dddbefc0f4398a..eb628a0f35cf23 100644 --- a/arch/x86/net/Makefile +++ b/arch/x86/net/Makefile @@ -8,3 +8,6 @@ ifeq ($(CONFIG_X86_32),y) else obj-$(CONFIG_BPF_JIT) += bpf_jit_comp.o bpf_timed_may_goto.o endif + +obj-y += rex.o +obj-y += rex_$(BITS).o diff --git a/arch/x86/net/rex.c b/arch/x86/net/rex.c new file mode 100644 index 00000000000000..287af85c45867e --- /dev/null +++ b/arch/x86/net/rex.c @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * X86-specific code for Rex support + */ +#define pr_fmt(fmt) "rex: " fmt + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/* Align to page size, since the stack trace is broken anyway */ +struct rex_stack { + char stack[REX_STACK_SIZE]; +} __aligned(PAGE_SIZE); + +DEFINE_PER_CPU_PAGE_ALIGNED(struct rex_stack, rex_stack_backing_store) +__visible; +DEFINE_PER_CPU(void *, rex_stack_ptr); + +DECLARE_PER_CPU(const struct bpf_prog *, rex_curr_prog); + +/* + * Not supposed to be called by other kernel code, therefore keep prototype + * private + */ +void rex_landingpad(void) __noreturn; + +static int map_rex_stack(unsigned int cpu) +{ + char *stack = (char *)per_cpu_ptr(&rex_stack_backing_store, cpu); + struct page *pages[REX_STACK_SIZE / PAGE_SIZE]; + void *va; + int i; + + for (i = 0; i < REX_STACK_SIZE / PAGE_SIZE; i++) { + phys_addr_t pa = per_cpu_ptr_to_phys(stack + (i << PAGE_SHIFT)); + + pages[i] = pfn_to_page(pa >> PAGE_SHIFT); + } + + va = vmap(pages, REX_STACK_SIZE / PAGE_SIZE, VM_MAP, PAGE_KERNEL); + if (!va) + return -ENOMEM; + + /* Store actual TOS to avoid adjustment in the hotpath */ + per_cpu(rex_stack_ptr, cpu) = va + REX_STACK_SIZE - 8; + + pr_info("Initialize rex_stack on CPU %d at 0x%llx\n", cpu, + ((u64)va) + REX_STACK_SIZE); + + return 0; +} + +int arch_init_rex_stack(void) +{ + int i, ret = 0; + for_each_online_cpu(i) { + ret = map_rex_stack(i); + if (ret < 0) { + pr_err("Failed to initialize rex stack on CPU %d\n", i); + break; + } + } + return ret; +} + +/* Do not declare rex_landingpad_asm() in header file since it should only be + * called from rex_landingpad() + */ +asmlinkage void __noreturn rex_landingpad_asm(void); + +void __noreturn rex_landingpad(void) +{ + struct task_struct *loader; + DEFINE_RATELIMIT_STATE(rex_rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + + /* Report error */ + if (__ratelimit(&rex_rs)) { + pr_err("%s\n", this_cpu_ptr(rex_log_buf)); + dump_stack(); + } + + loader = find_task_by_pid_ns( + this_cpu_read_stable(rex_curr_prog)->saved_state->loader_pid, + &init_pid_ns); + + /* Reuse the seccomp signal for now */ + if (loader) + force_sig_fault_to_task(SIGSYS, SYS_SECCOMP, NULL, loader); + + /* Reset the rex_termination_state set in rex panic handler */ + this_cpu_write(rex_termination_state, 0); + + /* Handle the rest fixups */ + rex_landingpad_asm(); +} diff --git a/arch/x86/net/rex_64.S b/arch/x86/net/rex_64.S new file mode 100644 index 00000000000000..44c730d63e4646 --- /dev/null +++ b/arch/x86/net/rex_64.S @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * X86 asm for Rex support + */ +#include +#include +#include +#include +#include + + .code64 + .section .text, "ax" + +/* + * Dispatcher func for Rex + * + * %rdi: ctx argument of the Rex program + * %rsi: pointer to the Rex program struct + * %rdx: fucntion pointer to the entry of the Rex program + */ +SYM_FUNC_START(rex_dispatcher_func) + /* save the callee-saved registers and the frame pointer*/ + pushq %rbp + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + + /* switch stack and save old rsp*/ + movq PER_CPU_VAR(rex_stack_ptr), %rbp + movq %rsp, (%rbp) + movq %rbp, %rsp + + /* record start time */ + movq jiffies(%rip), %r11 + movq %r11, PER_CPU_VAR(rex_prog_start_time) + + /* let the timer know we are in */ + movq %rsi, PER_CPU_VAR(rex_curr_prog) + + /* invoke bpf func */ + CALL_NOSPEC rdx + +/* + * Exit path: rex_landingpad also redirects the control flow here + * + * %rax: program return value or default return value in case of a panic + * %rsp: top entry of the Rex stack + */ +SYM_INNER_LABEL(rex_exit, SYM_L_GLOBAL) + /* let the timer know we are out */ + movq $0, PER_CPU_VAR(rex_curr_prog) + + /* pop old stack pointer into rsp */ + popq %rsp + + /* restore the callee-saved registers and the frame pointer */ + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx + popq %rbp + + /* return */ + RET +SYM_FUNC_END(rex_dispatcher_func) +EXPORT_SYMBOL(rex_dispatcher_func) + +/* + * Low-level fixups for Rust panics + */ +SYM_FUNC_START(rex_landingpad_asm) + /* set an return value of 0 */ + movq $0, %rax + + /* reset stack */ + movq PER_CPU_VAR(rex_stack_ptr), %rsp + + /* jump to exit path */ + jmp rex_exit +SYM_FUNC_END(rex_landingpad_asm) +STACK_FRAME_NON_STANDARD(rex_landingpad_asm); diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 05f92c812ac882..75be75678c9777 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -253,7 +253,7 @@ void xen_send_IPI_allbutself(int vector) static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) { - generic_smp_call_function_interrupt(); + generic_smp_call_function_interrupt(dev_id); inc_irq_stat(irq_call_count); return IRQ_HANDLED; @@ -261,7 +261,7 @@ static irqreturn_t xen_call_function_interrupt(int irq, void *dev_id) static irqreturn_t xen_call_function_single_interrupt(int irq, void *dev_id) { - generic_smp_call_function_single_interrupt(); + generic_smp_call_function_single_interrupt(dev_id); inc_irq_stat(irq_call_count); return IRQ_HANDLED; diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 35b1e25bd10437..b9ca45124c9ea8 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -33,6 +33,8 @@ #include #include +#include + struct bpf_verifier_env; struct bpf_verifier_log; struct perf_event; @@ -1399,6 +1401,7 @@ static __always_inline __bpfcall unsigned int bpf_dispatcher_nop_func( const struct bpf_insn *insnsi, bpf_func_t bpf_func) { + // printk(KERN_WARNING "DJW CALLING bpf_func at %p %d\n", bpf_func, __LINE__); return bpf_func(ctx, insnsi); } @@ -1714,7 +1717,13 @@ struct bpf_prog_aux { #ifdef CONFIG_FINEIBT struct bpf_ksym ksym_prefix; #endif - struct bpf_ksym ksym; + union { + struct bpf_ksym ksym; + struct { + struct bpf_ksym *rex_syms; + u64 nr_syms; + }; + }; const struct bpf_prog_ops *ops; const struct bpf_struct_ops *st_ops; struct bpf_map **used_maps; @@ -1772,6 +1781,30 @@ struct bpf_prog_aux { #define BPF_NR_CONTEXTS 4 /* normal, softirq, hardirq, NMI */ +struct rex_mem { + void *mem; + u32 total_page; +}; + +struct rex_saved_states{ + int cpu_id; + int loader_pid; + u64 unwinder_insn_off; + struct bpf_link *link; +}; + +struct rex_mem { + void *mem; + u32 total_page; +}; + +struct rex_saved_states{ + int cpu_id; + int loader_pid; + u64 unwinder_insn_off; + struct bpf_link *link; +}; + struct bpf_prog { u16 pages; /* Number of allocated pages */ u16 jited:1, /* Is our filter JIT'ed? */ @@ -1789,7 +1822,8 @@ struct bpf_prog { call_get_func_ip:1, /* Do we call get_func_ip() */ call_session_cookie:1, /* Do we call bpf_session_cookie() */ tstamp_type_access:1, /* Accessed __sk_buff->tstamp_type */ - sleepable:1; /* BPF program is sleepable */ + sleepable:1, /* BPF program is sleepable */ + no_bpf:1; enum bpf_prog_type type; /* Type of BPF program */ enum bpf_attach_type expected_attach_type; /* For some prog types */ u32 len; /* Number of filter blocks */ @@ -1804,6 +1838,9 @@ struct bpf_prog { const struct bpf_insn *insn); struct bpf_prog_aux *aux; /* Auxiliary fields */ struct sock_fprog_kern *orig_prog; /* Original BPF program */ + struct rex_mem mem; /* Rex base program pages */ + struct bpf_prog *base; /* Rex base program */ + struct rex_saved_states *saved_state; /* Instructions for interpreter */ union { DECLARE_FLEX_ARRAY(struct sock_filter, insns); @@ -3916,6 +3953,9 @@ int bpf_stream_stage_dump_stack(struct bpf_stream_stage *ss); bpf_stream_stage_free(&ss); \ }) +DECLARE_PER_CPU(char[MAX_BPRINTF_BUF], rex_log_buf); +void rex_trace_printk(void); + #ifdef CONFIG_BPF_LSM void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype); void bpf_cgroup_atype_put(int cgroup_atype); diff --git a/include/linux/filter.h b/include/linux/filter.h index 44d7ae95ddbccd..fbb8899496e685 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -24,6 +24,8 @@ #include #include +#include + #include #include @@ -685,6 +687,10 @@ struct sk_filter { struct bpf_prog *prog; }; +/* handler for termination requests */ +void rex_terminate(const struct bpf_prog *prog); +DECLARE_PER_CPU(const struct bpf_prog *, rex_curr_prog); + DECLARE_STATIC_KEY_FALSE(bpf_stats_enabled_key); extern struct mutex nf_conn_btf_access_lock; @@ -698,8 +704,8 @@ typedef unsigned int (*bpf_dispatcher_fn)(const void *ctx, const struct bpf_insn *)); static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog, - const void *ctx, - bpf_dispatcher_fn dfunc) + const void *ctx, + bpf_dispatcher_fn dfunc) { u32 ret; @@ -720,14 +726,59 @@ static __always_inline u32 __bpf_prog_run(const struct bpf_prog *prog, u64_stats_update_end_irqrestore(&stats->syncp, flags); } } else { + /* volatile u64 initial_time, completed_time; */ + /* initial_time = ktime_get_mono_fast_ns(); */ ret = dfunc(ctx, prog->insnsi, prog->bpf_func); + /* completed_time = ktime_get_mono_fast_ns(); */ + /* barrier(); */ + /* if (prog->type == BPF_PROG_TYPE_KPROBE) */ + /* printk("BPF dispatcher function overhead: %llu\n", completed_time - initial_time); */ + } + return ret; +} + + +typedef unsigned int (*rex_dispatcher_fn)(const void *ctx, + const struct bpf_prog *prog, + unsigned int (*bpf_func)(const void *, + const struct bpf_insn *)); + +static __always_inline u32 __rex_prog_run(const struct bpf_prog *prog, + const void *ctx, + rex_dispatcher_fn dfunc) +{ + u32 ret; + + cant_migrate(); + if (static_branch_unlikely(&bpf_stats_enabled_key)) { + struct bpf_prog_stats *stats; + u64 duration, start = sched_clock(); + unsigned long flags; + + ret = dfunc(ctx, prog, prog->bpf_func); + + duration = sched_clock() - start; + stats = this_cpu_ptr(prog->stats); + flags = u64_stats_update_begin_irqsave(&stats->syncp); + u64_stats_inc(&stats->cnt); + u64_stats_add(&stats->nsecs, duration); + u64_stats_update_end_irqrestore(&stats->syncp, flags); + } else { + /* volatile u64 initial_time, completed_time; */ + /* initial_time = ktime_get_mono_fast_ns(); */ + ret = dfunc(ctx, prog, prog->bpf_func); + /* completed_time = ktime_get_mono_fast_ns(); */ + /* barrier(); */ + /* if (prog->type == BPF_PROG_TYPE_KPROBE) */ + /* printk("BPF dispatcher function overhead: %llu\n", completed_time - initial_time); */ } return ret; } static __always_inline u32 bpf_prog_run(const struct bpf_prog *prog, const void *ctx) { - return __bpf_prog_run(prog, ctx, bpf_dispatcher_nop_func); + return prog->no_bpf ? __rex_prog_run(prog, ctx, rex_dispatcher_func) + : __bpf_prog_run(prog, ctx, bpf_dispatcher_nop_func); } /* @@ -1386,6 +1437,9 @@ struct bpf_prog *bpf_prog_ksym_find(unsigned long addr); void bpf_prog_kallsyms_add(struct bpf_prog *fp); void bpf_prog_kallsyms_del(struct bpf_prog *fp); +void rex_prog_kallsyms_add(struct bpf_prog *fp); +void rex_prog_kallsyms_del(struct bpf_prog *fp); + #else /* CONFIG_BPF_JIT */ static inline bool ebpf_jit_enabled(void) @@ -1451,6 +1505,14 @@ static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp) { } +static inline void rex_prog_kallsyms_add(struct bpf_prog *fp) +{ +} + +static inline void rex_prog_kallsyms_del(struct bpf_prog *fp) +{ +} + #endif /* CONFIG_BPF_JIT */ void bpf_prog_kallsyms_del_all(struct bpf_prog *fp); diff --git a/include/linux/smp.h b/include/linux/smp.h index 1ebd88026119a8..fb055d778288d3 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -26,6 +26,11 @@ struct __call_single_data { void *info; }; +struct termination_data { + struct bpf_prog *prog; + struct pt_regs *regs; +}; + #define CSD_INIT(_func, _info) \ (struct __call_single_data){ .func = (_func), .info = (_info), } @@ -175,7 +180,7 @@ bool cpus_peek_for_pending_ipi(const struct cpumask *mask); * Generic and arch helpers */ void __init call_function_init(void); -void generic_smp_call_function_single_interrupt(void); +void generic_smp_call_function_single_interrupt(struct pt_regs* regs); #define generic_smp_call_function_interrupt \ generic_smp_call_function_single_interrupt diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 02bd6ddb627821..3cc75896af6e5b 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -1178,8 +1178,8 @@ asmlinkage long sys_geteuid16(void); asmlinkage long sys_getgid16(void); asmlinkage long sys_getegid16(void); #endif - /* obsolete */ +asmlinkage long sys_hello(void); asmlinkage long sys_socketcall(int call, unsigned long __user *args); /* obsolete */ @@ -1334,3 +1334,4 @@ int __sys_getsockopt(int fd, int level, int optname, char __user *optval, int __sys_setsockopt(int fd, int level, int optname, char __user *optval, int optlen); #endif + diff --git a/include/net/xdp.h b/include/net/xdp.h index aa742f413c3585..51e4c96c2da788 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -687,13 +687,14 @@ static inline void xdp_clear_features_flag(struct net_device *dev) } static __always_inline u32 bpf_prog_run_xdp(const struct bpf_prog *prog, - struct xdp_buff *xdp) + struct xdp_buff *xdp) { /* Driver XDP hooks are invoked within a single NAPI poll cycle and thus * under local_bh_disable(), which provides the needed RCU protection * for accessing map entries. */ - u32 act = __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp)); + u32 act = prog->no_bpf ? __rex_prog_run(prog, xdp, rex_dispatcher_func) + : __bpf_prog_run(prog, xdp, BPF_DISPATCHER_FUNC(xdp)); if (static_branch_unlikely(&bpf_master_redirect_enabled_key)) { if (act == XDP_TX && netif_is_bond_slave(xdp->rxq->dev)) diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index c8d400b7680a81..15b58a94c1d1e3 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -992,7 +992,12 @@ enum bpf_cmd { BPF_PROG_BIND_MAP, BPF_TOKEN_CREATE, BPF_PROG_STREAM_READ_BY_FD, + BPF_PROG_LOAD_REX_BASE, + BPF_PROG_LOAD_REX, + BPF_SCHED_EXT_ATTACH_REX, + BPF_SCHED_EXT_DETACH_REX, BPF_PROG_ASSOC_STRUCT_OPS, + BPF_PROG_TERMINATE, __MAX_BPF_CMD, }; @@ -1091,6 +1096,7 @@ enum bpf_prog_type { BPF_PROG_TYPE_SK_LOOKUP, BPF_PROG_TYPE_SYSCALL, /* a program that can execute syscalls */ BPF_PROG_TYPE_NETFILTER, + BPF_PROG_TYPE_REX_BASE, __MAX_BPF_PROG_TYPE }; @@ -1507,6 +1513,28 @@ enum { BPF_STREAM_STDERR = 2, }; +struct rex_rela_dyn { + __u64 offset; + __u64 info; + __u64 addend; +}; + +struct rex_dyn_sym { + __u64 offset; + const char __user *symbol; +}; + +struct rex_text_sym { + __u64 offset; + __u64 size; + const char __user *symbol; +}; + +struct rex_sched_ops_sym { + const char __user *name; + __u64 offset; +}; + union bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ @@ -1595,6 +1623,7 @@ union bpf_attr { __u32 prog_flags; char prog_name[BPF_OBJ_NAME_LEN]; __u32 prog_ifindex; /* ifindex of netdev to prep for */ + __u64 unwinder_insn_off; /* For some prog types expected attach type must be known at * load time to verify attach type specific parts of prog * (context accesses, allowed helpers, etc). @@ -1615,6 +1644,23 @@ union bpf_attr { __u32 attach_btf_obj_fd; }; __u32 core_relo_cnt; /* number of bpf_core_relo */ + union { + struct { + __aligned_u64 map_offs; /* offsets of map relocs */ + __aligned_u64 dyn_relas; /* ptr to dynamic rela info */ + __aligned_u64 nr_dyn_relas; /* nr of dyn rela entries */ + __aligned_u64 dyn_syms; /* ptr to dyn sym entries */ + __aligned_u64 nr_dyn_syms; /* nr of dyn sym entries */ + __aligned_u64 text_syms; /* ptr to text sym info entries */ + __aligned_u64 nr_text_syms; /* nr of text sym info entries */ + __u32 rustfd; /* file descriptor of Rust Program */ + __u32 map_cnt; /* length map reloc array */ + }; + struct { + __aligned_u64 prog_offset; /* offset of prog in base */ + __u32 base_prog_fd; /* fd of the base prog */ + }; + }; __aligned_u64 fd_array; /* array of FDs */ __aligned_u64 core_relos; __u32 core_relo_rec_size; /* sizeof(struct bpf_core_relo) */ @@ -1916,6 +1962,17 @@ union bpf_attr { __u32 prog_fd; } prog_stream_read; + struct { /* BPF_SCHED_EXT_ATTACH_REX */ + __u32 base_prog_fd; + __aligned_u64 sched_ops_syms; /* ptr to rex_sched_ops_sym array */ + __u32 nr_sched_ops_syms; + __u32 timeout_ms; /* ops.timeout_ms, 0 = default */ + __u32 exit_dump_len; /* ops.exit_dump_len, 0 = default */ + __u32 pad; + __aligned_u64 ops_flags; /* SCX_OPS_* flags */ + char name[128]; /* ops.name; empty string = use base->aux->name */ + } sched_ext_attach; + struct { __u32 map_fd; __u32 prog_fd; diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile index 79cf22860a99ba..eb1efb2a6ea04f 100644 --- a/kernel/bpf/Makefile +++ b/kernel/bpf/Makefile @@ -67,6 +67,8 @@ ifeq ($(CONFIG_DMA_SHARED_BUFFER),y) obj-$(CONFIG_BPF_SYSCALL) += dmabuf_iter.o endif +obj-$(CONFIG_BPF_SYSCALL) += rex.o + CFLAGS_REMOVE_percpu_freelist.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_bpf_lru_list.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_queue_stack_maps.o = $(CC_FLAGS_FTRACE) diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index 7b675a451ec8ef..b6808fe44044b0 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -102,15 +102,26 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag gfp_t gfp_flags = bpf_memcg_flags(GFP_KERNEL | __GFP_ZERO | gfp_extra_flags); struct bpf_prog_aux *aux; struct bpf_prog *fp; + struct rex_saved_states *saved_state; // for BPF termination size = round_up(size, __PAGE_SIZE); fp = __vmalloc(size, gfp_flags); if (fp == NULL) return NULL; + //malloc(sizeof(*saved_state),GFP_KERNEL_ACCOUNT | gfp_extra_flags); + saved_state = + kzalloc(sizeof(*saved_state), + bpf_memcg_flags(GFP_KERNEL_ACCOUNT | gfp_extra_flags)); + if(saved_state == NULL){ + vfree(fp); + return NULL; + } + saved_state->cpu_id = -1; aux = kzalloc_obj(*aux, bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags)); if (aux == NULL) { vfree(fp); + kfree(saved_state); return NULL; } fp->active = __alloc_percpu_gfp(sizeof(u8[BPF_NR_CONTEXTS]), 4, @@ -118,6 +129,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag if (!fp->active) { vfree(fp); kfree(aux); + kfree(saved_state); return NULL; } @@ -125,6 +137,7 @@ struct bpf_prog *bpf_prog_alloc_no_stats(unsigned int size, gfp_t gfp_extra_flag fp->aux = aux; fp->aux->main_prog_aux = aux; fp->aux->prog = fp; + fp->saved_state = saved_state; fp->jit_requested = ebpf_jit_enabled(); fp->blinding_requested = bpf_jit_blinding_enabled(fp); #ifdef CONFIG_CGROUP_BPF @@ -292,6 +305,7 @@ void __bpf_prog_free(struct bpf_prog *fp) mutex_destroy(&fp->aux->st_ops_assoc_mutex); kfree(fp->aux->poke_tab); kfree(fp->aux); + kfree(fp->saved_state); } free_percpu(fp->stats); free_percpu(fp->active); @@ -709,6 +723,20 @@ void bpf_prog_kallsyms_del(struct bpf_prog *fp) #endif } +void rex_prog_kallsyms_add(struct bpf_prog *fp) +{ + for (int i = 0; i < fp->aux->nr_syms; i++) { + bpf_ksym_add(&fp->aux->rex_syms[i]); + } +} + +void rex_prog_kallsyms_del(struct bpf_prog *fp) +{ + for (int i = 0; i < fp->aux->nr_syms; i++) { + bpf_ksym_del(&fp->aux->rex_syms[i]); + } +} + static struct bpf_ksym *bpf_ksym_find(unsigned long addr) { struct latch_tree_node *n; @@ -1234,7 +1262,7 @@ bpf_jit_binary_hdr(const struct bpf_prog *fp) */ void __weak bpf_jit_free(struct bpf_prog *fp) { - if (fp->jited) { + if (fp->jited && !fp->no_bpf) { struct bpf_binary_header *hdr = bpf_jit_binary_hdr(fp); bpf_jit_binary_free(hdr); @@ -2961,6 +2989,22 @@ static void bpf_prog_free_deferred(struct work_struct *work) aux->func[i]->aux->poke_tab = NULL; bpf_jit_free(aux->func[i]); } + + if (aux->prog->no_bpf) { + BUG_ON(aux->func_cnt); + + if (aux->prog->base) { + bpf_prog_put(aux->prog->base); + } else { + set_memory_nx((unsigned long)aux->prog->mem.mem, aux->prog->mem.total_page); + set_memory_rw((unsigned long)aux->prog->mem.mem, aux->prog->mem.total_page); + vfree(aux->prog->mem.mem); + } + + // We have already cleared the prog pages + aux->prog->jited = 0; + } + if (aux->real_func_cnt) { kfree(aux->func); bpf_prog_unlock_free(aux->prog); @@ -3392,3 +3436,13 @@ struct bpf_prog *bpf_prog_find_from_stack(void) } #endif + +struct rex_cleanup_entry { + u64 valid; + void *cleanup_fn; + void *cleanup_arg; +}; + +#define IU_CLEANUP_ENTRIES_SIZE 64 + +DEFINE_PER_CPU(struct rex_cleanup_entry[64], rex_cleanup_entries) ____cacheline_aligned = { 0 }; diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 6eb6c82ed2ee1a..ab635caa30acbb 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -229,6 +229,8 @@ BPF_CALL_0(bpf_get_current_pid_tgid) if (unlikely(!task)) return -EINVAL; + //printk(KERN_WARNING "DJW bpf_get_current_pid_tgid %d [%llx]\n", __LINE__, (u64) task->tgid << 32 | task->pid); + return (u64) task->tgid << 32 | task->pid; } diff --git a/kernel/bpf/rex.c b/kernel/bpf/rex.c new file mode 100644 index 00000000000000..ac18f970bcb3b9 --- /dev/null +++ b/kernel/bpf/rex.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Architecture-independent code for Rex support + */ +#define pr_fmt(fmt) "rex: " fmt + +#include +#include + +#include + +/* Per-cpu log buffer to format printk and panic messages */ +DEFINE_PER_CPU(char[MAX_BPRINTF_BUF], rex_log_buf) = { 0 }; + +/* Set watchdog period to 20s */ +#define WATCHDOG_PERIOD_MS 20000U + +/* used by rex_terminate to check for BPF's IP before issuing termination */ +DEFINE_PER_CPU(unsigned char, rex_termination_state); + +/* Keeps track of prog start time */ +DEFINE_PER_CPU(unsigned long, rex_prog_start_time); + +/* Current program on this CPU */ +DEFINE_PER_CPU(const struct bpf_prog *, rex_curr_prog); +EXPORT_SYMBOL(rex_curr_prog); + +DEFINE_PER_CPU(struct hrtimer, rex_timer); + +static void check_running_progs(void) +{ + unsigned long start_time; + const struct bpf_prog *prog = this_cpu_read_stable(rex_curr_prog); + + /* Program not running on this CPU */ + if (!prog || !prog->no_bpf) + return; + + start_time = this_cpu_read_stable(rex_prog_start_time); + + /* Not reaching timeout */ + if (time_is_after_jiffies(start_time + + msecs_to_jiffies(WATCHDOG_PERIOD_MS))) + return; + + /* The program times out */ + rex_terminate(prog); +} + +void rex_terminate(const struct bpf_prog *prog) +{ + struct pt_regs *regs; + int prog_id; + + /* The termination handler is only supposed to be called in hardirq */ + WARN_ON(!in_hardirq()); + + regs = get_irq_regs(); + + /* We interrupted something that is not a rex program, probably some other softirq */ + if (!arch_on_rex_stack(regs)) { + this_cpu_write(rex_termination_state, 2); + return; + } + + prog_id = prog->aux->id; + pr_warn("Rex_terminate invoked for prog:%d\n", prog_id); + + if (this_cpu_read_stable(rex_termination_state) == 0) { + pr_warn("Program not in any helper/panic.\n"); + regs->ip = prog->saved_state->unwinder_insn_off; + } else { + pr_warn("Program in helper/panic.\n"); + this_cpu_write(rex_termination_state, 2); + } +} + +static enum hrtimer_restart timer_callback(struct hrtimer *timer) +{ + // pr_info("Rex_watchdog triggered\n"); + + check_running_progs(); + + /* Restart the timer */ + hrtimer_forward_now(timer, ms_to_ktime(WATCHDOG_PERIOD_MS)); + + /* Return HRTIMER_NORESTART to stop the timer */ + return HRTIMER_RESTART; +} + +static void start_timer_on_cpu(void *data __always_unused) +{ + struct hrtimer *local_timer = this_cpu_ptr(&rex_timer); + + hrtimer_setup(local_timer, timer_callback, CLOCK_MONOTONIC, + HRTIMER_MODE_REL_PINNED); + + /* boot the timer */ + hrtimer_start(local_timer, ms_to_ktime(WATCHDOG_PERIOD_MS), + HRTIMER_MODE_REL_PINNED); + + pr_info("Initialize time func on cpu %d\n", smp_processor_id()); + return; +} + +static int init_rex_watchdog(void) +{ + int i, ret; + pr_info("Initialize rex_watchdog\n"); + + for_each_online_cpu(i) { + ret = smp_call_function_single(i, start_timer_on_cpu, NULL, + true); + if (ret) { + pr_err("Failed to start timer on CPU %d\n", i); + return ret; + } + } + + return 0; +} + +static int __init init_rex(void) +{ + int ret = arch_init_rex_stack(); + return ret ?: init_rex_watchdog(); +} + +module_init(init_rex); diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c index 700938782bed2f..429dee647d7d2e 100644 --- a/kernel/bpf/syscall.c +++ b/kernel/bpf/syscall.c @@ -2,6 +2,7 @@ /* Copyright (c) 2011-2014 PLUMgrid, http://plumgrid.com */ #include +#include "asm-generic/set_memory.h" #include #include #include @@ -46,6 +47,13 @@ #include #include +#include + +#include +#include +#include // for show_regs +#include // for msleep + #define IS_FD_ARRAY(map) ((map)->map_type == BPF_MAP_TYPE_PERF_EVENT_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_CGROUP_ARRAY || \ (map)->map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS) @@ -111,6 +119,9 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, return res ? 0 : -E2BIG; } +static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred); +static void bpf_perf_link_release(struct bpf_link *link); + const struct bpf_map_ops bpf_map_offload_ops = { .map_meta_equal = bpf_map_meta_equal, .map_alloc = bpf_map_offload_map_alloc, @@ -2280,6 +2291,11 @@ static int find_prog_type(enum bpf_prog_type type, struct bpf_prog *prog) { const struct bpf_prog_ops *ops; + if (type == BPF_PROG_TYPE_REX_BASE) { + prog->type = type; + return 0; + } + if (type >= ARRAY_SIZE(bpf_prog_types)) return -EINVAL; type = array_index_nospec(type, ARRAY_SIZE(bpf_prog_types)); @@ -2375,13 +2391,20 @@ static void __bpf_prog_put_rcu(struct rcu_head *rcu) static void __bpf_prog_put_noref(struct bpf_prog *prog, bool deferred) { - bpf_prog_kallsyms_del_all(prog); + if (prog->no_bpf) { + rex_prog_kallsyms_del(prog); + kfree(prog->aux->rex_syms); + } else { + bpf_prog_kallsyms_del_all(prog); + } + btf_put(prog->aux->btf); module_put(prog->aux->mod); kvfree(prog->aux->jited_linfo); kvfree(prog->aux->linfo); kfree(prog->aux->kfunc_tab); kfree(prog->aux->ctx_arg_info); + if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); @@ -2910,38 +2933,1586 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) } } - bpf_cap = bpf_token_capable(token, CAP_BPF); - err = -EPERM; + bpf_cap = bpf_token_capable(token, CAP_BPF); + err = -EPERM; + + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && + !bpf_cap) + goto put_token; + + /* Intent here is for unprivileged_bpf_disabled to block BPF program + * creation for unprivileged users; other actions depend + * on fd availability and access to bpffs, so are dependent on + * object creation success. Even with unprivileged BPF disabled, + * capability checks are still carried out for these + * and other operations. + */ + if (sysctl_unprivileged_bpf_disabled && !bpf_cap) + goto put_token; + + if (attr->insn_cnt == 0 || + attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) { + err = -E2BIG; + goto put_token; + } + if (type != BPF_PROG_TYPE_SOCKET_FILTER && + type != BPF_PROG_TYPE_CGROUP_SKB && + !bpf_cap) + goto put_token; + + if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN)) + goto put_token; + if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) + goto put_token; + + /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog + * or btf, we need to check which one it is + */ + if (attr->attach_prog_fd) { + dst_prog = bpf_prog_get(attr->attach_prog_fd); + if (IS_ERR(dst_prog)) { + dst_prog = NULL; + attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); + if (IS_ERR(attach_btf)) { + err = -EINVAL; + goto put_token; + } + if (!btf_is_kernel(attach_btf)) { + /* attaching through specifying bpf_prog's BTF + * objects directly might be supported eventually + */ + btf_put(attach_btf); + err = -ENOTSUPP; + goto put_token; + } + } + } else if (attr->attach_btf_id) { + /* fall back to vmlinux BTF, if BTF type ID is specified */ + attach_btf = bpf_get_btf_vmlinux(); + if (IS_ERR(attach_btf)) { + err = PTR_ERR(attach_btf); + goto put_token; + } + if (!attach_btf) { + err = -EINVAL; + goto put_token; + } + btf_get(attach_btf); + } + + if (bpf_prog_load_check_attach(type, attr->expected_attach_type, + attach_btf, attr->attach_btf_id, + dst_prog)) { + if (dst_prog) + bpf_prog_put(dst_prog); + if (attach_btf) + btf_put(attach_btf); + err = -EINVAL; + goto put_token; + } + + /* plain bpf_prog allocation */ + prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); + if (!prog) { + if (dst_prog) + bpf_prog_put(dst_prog); + if (attach_btf) + btf_put(attach_btf); + err = -EINVAL; + goto put_token; + } + + prog->expected_attach_type = attr->expected_attach_type; + prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE); + prog->aux->attach_btf = attach_btf; + prog->aux->attach_btf_id = attr->attach_btf_id; + prog->aux->dst_prog = dst_prog; + prog->aux->dev_bound = !!attr->prog_ifindex; + prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; + + /* move token into prog->aux, reuse taken refcnt */ + prog->aux->token = token; + token = NULL; + + prog->aux->user = get_current_user(); + prog->len = attr->insn_cnt; + + err = -EFAULT; + if (copy_from_bpfptr(prog->insns, + make_bpfptr(attr->insns, uattr.is_kernel), + bpf_prog_insn_size(prog)) != 0) + goto free_prog; + /* copy eBPF program license from user space */ + if (strncpy_from_bpfptr(license, + make_bpfptr(attr->license, uattr.is_kernel), + sizeof(license) - 1) < 0) + goto free_prog; + license[sizeof(license) - 1] = 0; + + /* eBPF programs must be GPL compatible to use GPL-ed functions */ + prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; + + if (attr->signature) { + err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel); + if (err) + goto free_prog; + } + + prog->orig_prog = NULL; + prog->jited = 0; + prog->no_bpf = 0; + + atomic64_set(&prog->aux->refcnt, 1); + + if (bpf_prog_is_dev_bound(prog->aux)) { + err = bpf_prog_dev_bound_init(prog, attr); + if (err) + goto free_prog; + } + + if (type == BPF_PROG_TYPE_EXT && dst_prog && + bpf_prog_is_dev_bound(dst_prog->aux)) { + err = bpf_prog_dev_bound_inherit(prog, dst_prog); + if (err) + goto free_prog; + } + + /* + * Bookkeeping for managing the program attachment chain. + * + * It might be tempting to set attach_tracing_prog flag at the attachment + * time, but this will not prevent from loading bunch of tracing prog + * first, then attach them one to another. + * + * The flag attach_tracing_prog is set for the whole program lifecycle, and + * doesn't have to be cleared in bpf_tracing_link_release, since tracing + * programs cannot change attachment target. + */ + if (type == BPF_PROG_TYPE_TRACING && dst_prog && + dst_prog->type == BPF_PROG_TYPE_TRACING) { + prog->aux->attach_tracing_prog = true; + } + + /* find program type: socket_filter vs tracing_filter */ + err = find_prog_type(type, prog); + if (err < 0) + goto free_prog; + + prog->aux->load_time = ktime_get_boottime_ns(); + err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, + sizeof(attr->prog_name)); + if (err < 0) + goto free_prog; + + err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel); + if (err) + goto free_prog_sec; + + /* run eBPF verifier */ + err = bpf_check(&prog, attr, uattr, uattr_size); + if (err < 0) + goto free_used_maps; + + prog = bpf_prog_select_runtime(prog, &err); + if (err < 0) + goto free_used_maps; + + err = bpf_prog_mark_insn_arrays_ready(prog); + if (err < 0) + goto free_used_maps; + + err = bpf_prog_alloc_id(prog); + if (err) + goto free_used_maps; + + /* Upon success of bpf_prog_alloc_id(), the BPF prog is + * effectively publicly exposed. However, retrieving via + * bpf_prog_get_fd_by_id() will take another reference, + * therefore it cannot be gone underneath us. + * + * Only for the time /after/ successful bpf_prog_new_fd() + * and before returning to userspace, we might just hold + * one reference and any parallel close on that fd could + * rip everything out. Hence, below notifications must + * happen before bpf_prog_new_fd(). + * + * Also, any failure handling from this point onwards must + * be using bpf_prog_put() given the program is exposed. + */ + bpf_prog_kallsyms_add(prog); + perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); + bpf_audit_prog(prog, BPF_AUDIT_LOAD); + + err = bpf_prog_new_fd(prog); + if (err < 0) + bpf_prog_put(prog); + return err; + +free_used_maps: + /* In case we have subprogs, we need to wait for a grace + * period before we can tear down JIT memory since symbols + * are already exposed under kallsyms. + */ + __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); + return err; + +free_prog_sec: + security_bpf_prog_free(prog); +free_prog: + free_uid(prog->aux->user); + if (prog->aux->attach_btf) + btf_put(prog->aux->attach_btf); + bpf_prog_free(prog); +put_token: + bpf_token_put(token); + return err; +} + +static int bpf_prog_load_rex(union bpf_attr *attr, bpfptr_t uattr) +{ + enum bpf_prog_type type = attr->prog_type; + struct bpf_prog *prog, *dst_prog = NULL; + struct btf *attach_btf = NULL; + int err; + char license[128]; /* we don't support this for now */ + bool is_gpl; + struct bpf_prog *base; + + if (CHECK_ATTR(BPF_PROG_LOAD)) + return -EINVAL; + + if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | + BPF_F_ANY_ALIGNMENT | + BPF_F_TEST_STATE_FREQ | + BPF_F_SLEEPABLE | + BPF_F_TEST_RND_HI32)) + return -EINVAL; + + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && + !bpf_capable()) + return -EPERM; + + /* copy eBPF program license from user space */ + if (strncpy_from_bpfptr(license, + make_bpfptr(attr->license, uattr.is_kernel), + sizeof(license) - 1) < 0) + return -EFAULT; + license[sizeof(license) - 1] = 0; + + /* eBPF programs must be GPL compatible to use GPL-ed functions */ + is_gpl = license_is_gpl_compatible(license); + + if (!bpf_capable()) + return -EPERM; + + if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + if (is_perfmon_prog_type(type) && !perfmon_capable()) + return -EPERM; + + /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog + * or btf, we need to check which one it is + */ + if (attr->attach_prog_fd) { + dst_prog = bpf_prog_get(attr->attach_prog_fd); + if (IS_ERR(dst_prog)) { + dst_prog = NULL; + attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); + if (IS_ERR(attach_btf)) + return -EINVAL; + if (!btf_is_kernel(attach_btf)) { + /* attaching through specifying bpf_prog's BTF + * objects directly might be supported eventually + */ + btf_put(attach_btf); + return -ENOTSUPP; + } + } + } else if (attr->attach_btf_id) { + /* fall back to vmlinux BTF, if BTF type ID is specified */ + attach_btf = bpf_get_btf_vmlinux(); + if (IS_ERR(attach_btf)) + return PTR_ERR(attach_btf); + if (!attach_btf) + return -EINVAL; + btf_get(attach_btf); + } + + bpf_prog_load_fixup_attach_type(attr); + if (bpf_prog_load_check_attach(type, attr->expected_attach_type, + attach_btf, attr->attach_btf_id, + dst_prog)) { + if (dst_prog) + bpf_prog_put(dst_prog); + if (attach_btf) + btf_put(attach_btf); + return -EINVAL; + } + + /* plain bpf_prog allocation */ + prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); + if (!prog) { + if (dst_prog) + bpf_prog_put(dst_prog); + if (attach_btf) + btf_put(attach_btf); + return -ENOMEM; + } + + prog->expected_attach_type = attr->expected_attach_type; + prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE); + prog->aux->attach_btf = attach_btf; + prog->aux->attach_btf_id = attr->attach_btf_id; + prog->aux->dst_prog = dst_prog; + prog->aux->offload_requested = !!attr->prog_ifindex; + + prog->aux->user = get_current_user(); + prog->len = attr->insn_cnt; + + err = -EFAULT; + + prog->orig_prog = NULL; + prog->jited = 1; + + atomic64_set(&prog->aux->refcnt, 1); + prog->gpl_compatible = is_gpl ? 1 : 0; + + if (bpf_prog_is_dev_bound(prog->aux)) { + err = bpf_prog_dev_bound_init(prog, attr); + if (err) + goto free_prog_sec; + } + + /* find program type: socket_filter vs tracing_filter */ + err = find_prog_type(type, prog); + if (err < 0) + goto free_prog_sec; + + prog->aux->load_time = ktime_get_boottime_ns(); + err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, + sizeof(attr->prog_name)); + if (err < 0) + goto free_prog_sec; + + prog->no_bpf = 1; + + /* This gets the refcnt */ + base = bpf_prog_get(attr->base_prog_fd); + if (IS_ERR(base)) { + err = PTR_ERR(base); + goto free_used_maps; + } + + prog->base = base; + + if (attr->prog_offset >= base->mem.total_page << PAGE_SHIFT) { + err = -EINVAL; + goto free_base; + } + + prog->bpf_func = (void *)((u64)base->mem.mem + attr->prog_offset); + + /* Rust unwinder offset */ + prog->saved_state->unwinder_insn_off = + (u64)base->mem.mem + (u64)attr->unwinder_insn_off; + prog->saved_state->loader_pid = task_pid_nr(current); + + err = bpf_prog_alloc_id(prog); + if (err) + goto free_base; + + /* Upon success of bpf_prog_alloc_id(), the BPF prog is + * effectively publicly exposed. However, retrieving via + * bpf_prog_get_fd_by_id() will take another reference, + * therefore it cannot be gone underneath us. + * + * Only for the time /after/ successful bpf_prog_new_fd() + * and before returning to userspace, we might just hold + * one reference and any parallel close on that fd could + * rip everything out. Hence, below notifications must + * happen before bpf_prog_new_fd(). + * + * Also, any failure handling from this point onwards must + * be using bpf_prog_put() given the program is exposed. + */ + perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); + bpf_audit_prog(prog, BPF_AUDIT_LOAD); + + err = bpf_prog_new_fd(prog); + if (err < 0) + bpf_prog_put(prog); + return err; + +free_base: + prog->base = NULL; + bpf_prog_put(base); +free_used_maps: + /* In case we have subprogs, we need to wait for a grace + * period before we can tear down JIT memory since symbols + * are already exposed under kallsyms. + */ + __bpf_prog_put_noref(prog, prog->aux->func_cnt); + return err; +free_prog_sec: + free_uid(prog->aux->user); + security_bpf_prog_free(prog); +// free_prog: TODO: Needs to fix error path + if (prog->aux->attach_btf) + btf_put(prog->aux->attach_btf); + bpf_prog_free(prog); + return err; +} + +static unsigned int __rex_prog_empty(const void *ctx, + const struct bpf_insn *insn) +{ + return 0; +} + +/* + * Define EM_TARGET, EM_PAGE_SIZE and EI_DATA_TARGET for the architecture we + * are compiling on. + */ +#if defined(__x86_64__) +#define EM_TARGET EM_X86_64 +#define EM_PAGE_SIZE 0x1000 +#define EI_DATA_TARGET ELFDATA2LSB +#elif defined(__aarch64__) +#define EM_TARGET EM_AARCH64 +#define EM_PAGE_SIZE 0x1000 +#define EI_DATA_TARGET ELFDATA2LSB +#elif defined(__powerpc64__) +#define EM_TARGET EM_PPC64 +#define EM_PAGE_SIZE 0x10000 +#define EI_DATA_TARGET ELFDATA2MSB +#else +#error Unsupported target +#endif + +static bool ehdr_is_valid(const Elf64_Ehdr *hdr) +{ + /* + * 1. Validate that this is an ELF64 header we support. + * + * Note: e_ident[EI_OSABI] and e_ident[EI_ABIVERSION] are deliberately NOT + * checked as compilers do not provide a way to override this without + * building the entire toolchain from scratch. + */ + if (!(hdr->e_ident[EI_MAG0] == ELFMAG0 + && hdr->e_ident[EI_MAG1] == ELFMAG1 + && hdr->e_ident[EI_MAG2] == ELFMAG2 + && hdr->e_ident[EI_MAG3] == ELFMAG3 + && hdr->e_ident[EI_CLASS] == ELFCLASS64 + && hdr->e_ident[EI_DATA] == EI_DATA_TARGET + && hdr->e_version == EV_CURRENT)) + return false; + /* + * 2. Validate ELF64 header internal sizes match what we expect, and that + * at least one program header entry is present. + */ + if (hdr->e_ehsize != sizeof (Elf64_Ehdr)) + return false; + if (hdr->e_phnum < 1) + return false; + if (hdr->e_phentsize != sizeof (Elf64_Phdr)) + return false; + /* + * 3. Validate that this is an executable for our target architecture. + */ + if ((hdr->e_type != ET_EXEC) + && (hdr->e_type != ET_DYN)) /* DJW: PIE makes ET_DYN */ + return false; + if (hdr->e_machine != EM_TARGET) + return false; + + return true; +} + +/* + * Align (addr) down to (align) boundary. Returns 1 if (align) is not a + * non-zero power of 2. + */ +static int align_down(Elf64_Addr addr, Elf64_Xword align, + Elf64_Addr *out_result) +{ + if (align > 0 && (align & (align - 1)) == 0) { + *out_result = addr & -align; + return 0; + } + else + return 1; +} + +/* + * Align (addr) up to (align) boundary. Returns 1 if an overflow would occur or + * (align) is not a non-zero power of 2, otherwise result in (*out_result) and + * 0. + */ +static int align_up(Elf64_Addr addr, Elf64_Xword align, Elf64_Addr *out_result) +{ + Elf64_Addr result; + + if (align > 0 && (align & (align - 1)) == 0) { + if (check_add_overflow(addr, (align - 1), &result)) + return 1; + result = result & -align; + *out_result = result; + return 0; + } + else + return 1; +} + +static int elf_read(struct file *file, void *buf, size_t len, loff_t pos) +{ + ssize_t rv; + + rv = kernel_read(file, buf, len, &pos); + if (unlikely(rv != len)) { + return (rv < 0) ? rv : -EIO; + } + return 0; +} + +static int rex_parse_maps(union bpf_attr *attr, struct bpf_prog *prog, + u64 addr_start) +{ + u64 map_offs[MAX_USED_MAPS]; + struct bpf_map **used_maps; + int idx, ret = 0; + + if (attr->map_cnt >= MAX_USED_MAPS) + return -EINVAL; + + if (copy_from_bpfptr(map_offs, USER_BPFPTR((void *)(attr->map_offs)), + sizeof(u64) * attr->map_cnt) != 0) + return -EFAULT; + + used_maps = kmalloc(sizeof(*used_maps) * attr->map_cnt, GFP_KERNEL); + if (!used_maps) + return -ENOMEM; + + for (idx = 0; idx < attr->map_cnt; idx++) { + u64 *map_addr = (u64 *)(addr_start + map_offs[idx]); + struct bpf_map *curr = bpf_map_get(*map_addr); + unsigned int level; + pte_t *pte = lookup_address((unsigned long)map_addr, &level); + bool is_ro = !pte_write(*pte); + unsigned long start = (unsigned long)map_addr & PAGE_MASK; + unsigned long end = ((unsigned long)map_addr + sizeof(curr)) & + PAGE_MASK; + int nr_pages = start == end ? 1 : 2; + + if (IS_ERR(curr)) { + ret = PTR_ERR(curr); + goto free_used_maps; + } + + used_maps[idx] = curr; + + /* Maps might (or will always?) be in .data, which is read-only */ + if (is_ro) + set_memory_rw(start, nr_pages); + *map_addr = (u64)curr; + if (is_ro) + set_memory_ro(start, nr_pages); + } + prog->aux->used_maps = used_maps; + prog->aux->used_map_cnt = attr->map_cnt; + + return 0; + +free_used_maps: + kfree(used_maps); + return ret; +} + +static int rex_parse_relas(union bpf_attr *attr, u64 addr_start) +{ + int i = 0; + int ret = 0; + u64 relas_size = attr->nr_dyn_relas * sizeof(struct rex_rela_dyn); + struct rex_rela_dyn *relas = kmalloc_array(attr->nr_dyn_relas, + sizeof(*relas), GFP_KERNEL); + + if (!relas) + return -ENOMEM; + + if (copy_from_bpfptr(relas, USER_BPFPTR((void *)(attr->dyn_relas)), + relas_size) != 0) { + ret = -EFAULT; + goto free_relas; + } + + for (i = 0; i < attr->nr_dyn_relas; i++) { + u64 *abs_addr; + + if (ELF64_R_TYPE(relas[i].info) != R_X86_64_RELATIVE) { + ret = -EINVAL; + goto free_relas; + } + + abs_addr = (u64 *)(addr_start + relas[i].offset); + *abs_addr = addr_start + relas[i].addend; + } + +free_relas: + kfree(relas); + return ret; +} + +static int rex_parse_dyn_syms(union bpf_attr *attr, u64 addr_start, struct bpf_prog *prog) +{ + int i = 0, ret = 0; + u64 syms_size = attr->nr_dyn_syms * sizeof(struct rex_dyn_sym); + struct rex_dyn_sym *syms = kmalloc_array(attr->nr_dyn_syms, + sizeof(*syms), GFP_KERNEL); + char name[KSYM_NAME_LEN] = { 0 }; + + if (!syms) + return -ENOMEM; + + if (copy_from_bpfptr(syms, USER_BPFPTR((void *)attr->dyn_syms), + syms_size) != 0) { + ret = -EFAULT; + goto free_syms; + } + + for (i = 0; i < attr->nr_dyn_syms; i++) { + u64 *abs_addr = (u64 *)(addr_start + syms[i].offset); + u64 sym_addr; + + memset(name, 0, KSYM_NAME_LEN); + ret = strncpy_from_user(name, syms[i].symbol, KSYM_NAME_LEN); + if (ret == KSYM_NAME_LEN) + ret = -E2BIG; + if (ret < 0) + goto free_syms; + + sym_addr = kallsyms_lookup_name(name); + if (!sym_addr) { + ret = -EINVAL; + goto free_syms; + } + + /* A better way is to create a dedicated kprobe program type that can + * override return values */ + if (IS_ENABLED(CONFIG_BPF_KPROBE_OVERRIDE)) { + extern void just_return_func(void); + if (sym_addr == (u64)just_return_func) + prog->kprobe_override = 1; + } + + *abs_addr = sym_addr; + } + + ret = 0; + +free_syms: + kfree(syms); + return ret; +} + +static int rex_parse_text_syms(union bpf_attr *attr, u64 addr_start, + struct bpf_prog *prog) +{ + int ret = 0; + u64 syms_size = attr->nr_text_syms * sizeof(struct rex_text_sym); + char name[KSYM_NAME_LEN] = { 0 }; + struct rex_text_sym *text_syms = kmalloc_array( + attr->nr_text_syms, sizeof(*text_syms), GFP_KERNEL); + struct bpf_ksym *ksyms; + + if (!text_syms) + return -ENOMEM; + + ksyms = kmalloc_array(attr->nr_text_syms, sizeof(*ksyms), + GFP_KERNEL | __GFP_ZERO); + if (!ksyms) { + ret = -ENOMEM; + goto free_text_syms; + } + + if (copy_from_bpfptr(text_syms, USER_BPFPTR((void *)attr->text_syms), + syms_size) != 0) { + ret = -EFAULT; + goto free_ksyms; + } + + for (int i = 0; i < attr->nr_text_syms; i++) { + u64 abs_addr = addr_start + text_syms[i].offset; + char *sym = ksyms[i].name; + const char *end = sym + KSYM_NAME_LEN; + + memset(name, 0, KSYM_NAME_LEN); + ret = strncpy_from_user(name, text_syms[i].symbol, + KSYM_NAME_LEN); + if (ret == KSYM_NAME_LEN) + ret = -E2BIG; + if (ret < 0) + goto free_ksyms; + + ksyms[i].prog = true; + ksyms[i].start = abs_addr; + ksyms[i].end = abs_addr + text_syms[i].size; + + sym += snprintf(sym, KSYM_NAME_LEN, "rex_prog_"); + sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); + snprintf(sym, (size_t)(end - sym), "::%s", name); + + INIT_LIST_HEAD(&ksyms[i].lnode); + } + + prog->aux->rex_syms = ksyms; + prog->aux->nr_syms = attr->nr_text_syms; + ret = 0; + + /* Don't free ksyms on success as we have already given away ownership */ + goto free_text_syms; + +free_ksyms: + kfree(ksyms); +free_text_syms: + kfree(text_syms); + return ret; +} + +#define MAX_PROG_SZ (8192 << 4) +static int bpf_prog_load_rex_base(union bpf_attr *attr, bpfptr_t uattr) +{ + enum bpf_prog_type type = attr->prog_type; + struct bpf_prog *prog, *dst_prog = NULL; + struct btf *attach_btf = NULL; + int err; + char license[128]; + bool is_gpl; + + void *mem; + Elf64_Phdr *phdr = NULL; + Elf64_Ehdr *ehdr = NULL; + Elf64_Addr e_entry; /* Program entry point */ + Elf64_Addr e_end; /* Highest memory address occupied */ + struct file *filp; + size_t ph_size; + Elf64_Addr plast_vaddr = 0; + Elf64_Half ph_i; + u64 addr_start = 0; + int *vm_size = NULL, *sec_off = NULL; + int total_vm = 0; + + if (CHECK_ATTR(BPF_PROG_LOAD)) + return -EINVAL; + if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | + BPF_F_ANY_ALIGNMENT | + BPF_F_TEST_STATE_FREQ | + BPF_F_SLEEPABLE | + BPF_F_TEST_RND_HI32)) + return -EINVAL; + + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && + !bpf_cap) + goto put_token; + + /* Intent here is for unprivileged_bpf_disabled to block BPF program + * creation for unprivileged users; other actions depend + * on fd availability and access to bpffs, so are dependent on + * object creation success. Even with unprivileged BPF disabled, + * capability checks are still carried out for these + * and other operations. + */ + if (sysctl_unprivileged_bpf_disabled && !bpf_cap) + goto put_token; + + if (attr->insn_cnt == 0 || + attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) { + err = -E2BIG; + goto put_token; + } + if (type != BPF_PROG_TYPE_SOCKET_FILTER && + type != BPF_PROG_TYPE_CGROUP_SKB && + !bpf_cap) + goto put_token; + + if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN)) + goto put_token; + if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) + goto put_token; + + /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog + * or btf, we need to check which one it is + */ + if (attr->attach_prog_fd) { + dst_prog = bpf_prog_get(attr->attach_prog_fd); + if (IS_ERR(dst_prog)) { + dst_prog = NULL; + attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); + if (IS_ERR(attach_btf)) { + err = -EINVAL; + goto put_token; + } + if (!btf_is_kernel(attach_btf)) { + /* attaching through specifying bpf_prog's BTF + * objects directly might be supported eventually + */ + btf_put(attach_btf); + err = -ENOTSUPP; + goto put_token; + } + } + } else if (attr->attach_btf_id) { + /* fall back to vmlinux BTF, if BTF type ID is specified */ + attach_btf = bpf_get_btf_vmlinux(); + if (IS_ERR(attach_btf)) { + err = PTR_ERR(attach_btf); + goto put_token; + } + if (!attach_btf) { + err = -EINVAL; + goto put_token; + } + btf_get(attach_btf); + } + + if (bpf_prog_load_check_attach(type, attr->expected_attach_type, + attach_btf, attr->attach_btf_id, + dst_prog)) { + if (dst_prog) + bpf_prog_put(dst_prog); + if (attach_btf) + btf_put(attach_btf); + err = -EINVAL; + goto put_token; + } + + /* plain bpf_prog allocation */ + prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); + if (!prog) { + if (dst_prog) + bpf_prog_put(dst_prog); + if (attach_btf) + btf_put(attach_btf); + err = -EINVAL; + goto put_token; + } + + prog->expected_attach_type = attr->expected_attach_type; + prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE); + prog->aux->attach_btf = attach_btf; + prog->aux->attach_btf_id = attr->attach_btf_id; + prog->aux->dst_prog = dst_prog; + prog->aux->dev_bound = !!attr->prog_ifindex; + prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; + + /* move token into prog->aux, reuse taken refcnt */ + prog->aux->token = token; + token = NULL; + + prog->aux->user = get_current_user(); + prog->len = attr->insn_cnt; + + err = -EFAULT; + if (copy_from_bpfptr(prog->insns, + make_bpfptr(attr->insns, uattr.is_kernel), + bpf_prog_insn_size(prog)) != 0) + goto free_prog; + /* copy eBPF program license from user space */ + if (strncpy_from_bpfptr(license, + make_bpfptr(attr->license, uattr.is_kernel), + sizeof(license) - 1) < 0) + goto free_prog; + license[sizeof(license) - 1] = 0; + + /* eBPF programs must be GPL compatible to use GPL-ed functions */ + prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; + + if (attr->signature) { + err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel); + if (err) + goto free_prog; + } + + prog->orig_prog = NULL; + prog->jited = 0; + prog->no_bpf = 0; + + atomic64_set(&prog->aux->refcnt, 1); + + if (bpf_prog_is_dev_bound(prog->aux)) { + err = bpf_prog_dev_bound_init(prog, attr); + if (err) + goto free_prog; + } + + if (type == BPF_PROG_TYPE_EXT && dst_prog && + bpf_prog_is_dev_bound(dst_prog->aux)) { + err = bpf_prog_dev_bound_inherit(prog, dst_prog); + if (err) + goto free_prog; + } + + /* + * Bookkeeping for managing the program attachment chain. + * + * It might be tempting to set attach_tracing_prog flag at the attachment + * time, but this will not prevent from loading bunch of tracing prog + * first, then attach them one to another. + * + * The flag attach_tracing_prog is set for the whole program lifecycle, and + * doesn't have to be cleared in bpf_tracing_link_release, since tracing + * programs cannot change attachment target. + */ + if (type == BPF_PROG_TYPE_TRACING && dst_prog && + dst_prog->type == BPF_PROG_TYPE_TRACING) { + prog->aux->attach_tracing_prog = true; + } + + /* find program type: socket_filter vs tracing_filter */ + err = find_prog_type(type, prog); + if (err < 0) + goto free_prog; + + prog->aux->load_time = ktime_get_boottime_ns(); + err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, + sizeof(attr->prog_name)); + if (err < 0) + goto free_prog; + + err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel); + if (err) + goto free_prog_sec; + + /* run eBPF verifier */ + err = bpf_check(&prog, attr, uattr, uattr_size); + if (err < 0) + goto free_used_maps; + + prog = bpf_prog_select_runtime(prog, &err); + if (err < 0) + goto free_used_maps; + + err = bpf_prog_mark_insn_arrays_ready(prog); + if (err < 0) + goto free_used_maps; + + err = bpf_prog_alloc_id(prog); + if (err) + goto free_used_maps; + + /* Upon success of bpf_prog_alloc_id(), the BPF prog is + * effectively publicly exposed. However, retrieving via + * bpf_prog_get_fd_by_id() will take another reference, + * therefore it cannot be gone underneath us. + * + * Only for the time /after/ successful bpf_prog_new_fd() + * and before returning to userspace, we might just hold + * one reference and any parallel close on that fd could + * rip everything out. Hence, below notifications must + * happen before bpf_prog_new_fd(). + * + * Also, any failure handling from this point onwards must + * be using bpf_prog_put() given the program is exposed. + */ + bpf_prog_kallsyms_add(prog); + perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); + bpf_audit_prog(prog, BPF_AUDIT_LOAD); + + err = bpf_prog_new_fd(prog); + if (err < 0) + bpf_prog_put(prog); + return err; + +free_used_maps: + /* In case we have subprogs, we need to wait for a grace + * period before we can tear down JIT memory since symbols + * are already exposed under kallsyms. + */ + __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); + return err; + +free_prog_sec: + security_bpf_prog_free(prog); +free_prog: + free_uid(prog->aux->user); + if (prog->aux->attach_btf) + btf_put(prog->aux->attach_btf); + bpf_prog_free(prog); +put_token: + bpf_token_put(token); + return err; +} + +static int bpf_prog_load_rex(union bpf_attr *attr, bpfptr_t uattr) +{ + enum bpf_prog_type type = attr->prog_type; + struct bpf_prog *prog, *dst_prog = NULL; + struct btf *attach_btf = NULL; + int err; + char license[128]; /* we don't support this for now */ + bool is_gpl; + struct bpf_prog *base; + + if (CHECK_ATTR(BPF_PROG_LOAD)) + return -EINVAL; + + if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | + BPF_F_ANY_ALIGNMENT | + BPF_F_TEST_STATE_FREQ | + BPF_F_SLEEPABLE | + BPF_F_TEST_RND_HI32)) + return -EINVAL; + + if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && + (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && + !bpf_capable()) + return -EPERM; + + /* copy eBPF program license from user space */ + if (strncpy_from_bpfptr(license, + make_bpfptr(attr->license, uattr.is_kernel), + sizeof(license) - 1) < 0) + return -EFAULT; + license[sizeof(license) - 1] = 0; + + /* eBPF programs must be GPL compatible to use GPL-ed functions */ + is_gpl = license_is_gpl_compatible(license); + + if (!bpf_capable()) + return -EPERM; + + if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + if (is_perfmon_prog_type(type) && !perfmon_capable()) + return -EPERM; + + /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog + * or btf, we need to check which one it is + */ + if (attr->attach_prog_fd) { + dst_prog = bpf_prog_get(attr->attach_prog_fd); + if (IS_ERR(dst_prog)) { + dst_prog = NULL; + attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); + if (IS_ERR(attach_btf)) + return -EINVAL; + if (!btf_is_kernel(attach_btf)) { + /* attaching through specifying bpf_prog's BTF + * objects directly might be supported eventually + */ + btf_put(attach_btf); + return -ENOTSUPP; + } + } + } else if (attr->attach_btf_id) { + /* fall back to vmlinux BTF, if BTF type ID is specified */ + attach_btf = bpf_get_btf_vmlinux(); + if (IS_ERR(attach_btf)) + return PTR_ERR(attach_btf); + if (!attach_btf) + return -EINVAL; + btf_get(attach_btf); + } + + bpf_prog_load_fixup_attach_type(attr); + if (bpf_prog_load_check_attach(type, attr->expected_attach_type, + attach_btf, attr->attach_btf_id, + dst_prog)) { + if (dst_prog) + bpf_prog_put(dst_prog); + if (attach_btf) + btf_put(attach_btf); + return -EINVAL; + } + + /* plain bpf_prog allocation */ + prog = bpf_prog_alloc(bpf_prog_size(attr->insn_cnt), GFP_USER); + if (!prog) { + if (dst_prog) + bpf_prog_put(dst_prog); + if (attach_btf) + btf_put(attach_btf); + return -ENOMEM; + } + + prog->expected_attach_type = attr->expected_attach_type; + prog->sleepable = !!(attr->prog_flags & BPF_F_SLEEPABLE); + prog->aux->attach_btf = attach_btf; + prog->aux->attach_btf_id = attr->attach_btf_id; + prog->aux->dst_prog = dst_prog; + prog->aux->offload_requested = !!attr->prog_ifindex; + + prog->aux->user = get_current_user(); + prog->len = attr->insn_cnt; + + err = -EFAULT; + + prog->orig_prog = NULL; + prog->jited = 1; + + atomic64_set(&prog->aux->refcnt, 1); + prog->gpl_compatible = is_gpl ? 1 : 0; + + if (bpf_prog_is_dev_bound(prog->aux)) { + err = bpf_prog_dev_bound_init(prog, attr); + if (err) + goto free_prog_sec; + } + + /* find program type: socket_filter vs tracing_filter */ + err = find_prog_type(type, prog); + if (err < 0) + goto free_prog_sec; + + prog->aux->load_time = ktime_get_boottime_ns(); + err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, + sizeof(attr->prog_name)); + if (err < 0) + goto free_prog_sec; + + prog->no_bpf = 1; + + /* This gets the refcnt */ + base = bpf_prog_get(attr->base_prog_fd); + if (IS_ERR(base)) { + err = PTR_ERR(base); + goto free_used_maps; + } + + prog->base = base; + + if (attr->prog_offset >= base->mem.total_page << PAGE_SHIFT) { + err = -EINVAL; + goto free_base; + } + + prog->bpf_func = (void *)((u64)base->mem.mem + attr->prog_offset); + + /* Rust unwinder offset */ + prog->saved_state->unwinder_insn_off = + (u64)base->mem.mem + (u64)attr->unwinder_insn_off; + prog->saved_state->loader_pid = task_pid_nr(current); + + err = bpf_prog_alloc_id(prog); + if (err) + goto free_base; + + /* Upon success of bpf_prog_alloc_id(), the BPF prog is + * effectively publicly exposed. However, retrieving via + * bpf_prog_get_fd_by_id() will take another reference, + * therefore it cannot be gone underneath us. + * + * Only for the time /after/ successful bpf_prog_new_fd() + * and before returning to userspace, we might just hold + * one reference and any parallel close on that fd could + * rip everything out. Hence, below notifications must + * happen before bpf_prog_new_fd(). + * + * Also, any failure handling from this point onwards must + * be using bpf_prog_put() given the program is exposed. + */ + perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); + bpf_audit_prog(prog, BPF_AUDIT_LOAD); + + err = bpf_prog_new_fd(prog); + if (err < 0) + bpf_prog_put(prog); + return err; + +free_base: + prog->base = NULL; + bpf_prog_put(base); +free_used_maps: + /* In case we have subprogs, we need to wait for a grace + * period before we can tear down JIT memory since symbols + * are already exposed under kallsyms. + */ + __bpf_prog_put_noref(prog, prog->aux->func_cnt); + return err; +free_prog_sec: + free_uid(prog->aux->user); + security_bpf_prog_free(prog); +// free_prog: TODO: Needs to fix error path + if (prog->aux->attach_btf) + btf_put(prog->aux->attach_btf); + bpf_prog_free(prog); + return err; +} + +static unsigned int __rex_prog_empty(const void *ctx, + const struct bpf_insn *insn) +{ + return 0; +} + +/* + * Define EM_TARGET, EM_PAGE_SIZE and EI_DATA_TARGET for the architecture we + * are compiling on. + */ +#if defined(__x86_64__) +#define EM_TARGET EM_X86_64 +#define EM_PAGE_SIZE 0x1000 +#define EI_DATA_TARGET ELFDATA2LSB +#elif defined(__aarch64__) +#define EM_TARGET EM_AARCH64 +#define EM_PAGE_SIZE 0x1000 +#define EI_DATA_TARGET ELFDATA2LSB +#elif defined(__powerpc64__) +#define EM_TARGET EM_PPC64 +#define EM_PAGE_SIZE 0x10000 +#define EI_DATA_TARGET ELFDATA2MSB +#else +#error Unsupported target +#endif + +static bool ehdr_is_valid(const Elf64_Ehdr *hdr) +{ + /* + * 1. Validate that this is an ELF64 header we support. + * + * Note: e_ident[EI_OSABI] and e_ident[EI_ABIVERSION] are deliberately NOT + * checked as compilers do not provide a way to override this without + * building the entire toolchain from scratch. + */ + if (!(hdr->e_ident[EI_MAG0] == ELFMAG0 + && hdr->e_ident[EI_MAG1] == ELFMAG1 + && hdr->e_ident[EI_MAG2] == ELFMAG2 + && hdr->e_ident[EI_MAG3] == ELFMAG3 + && hdr->e_ident[EI_CLASS] == ELFCLASS64 + && hdr->e_ident[EI_DATA] == EI_DATA_TARGET + && hdr->e_version == EV_CURRENT)) + return false; + /* + * 2. Validate ELF64 header internal sizes match what we expect, and that + * at least one program header entry is present. + */ + if (hdr->e_ehsize != sizeof (Elf64_Ehdr)) + return false; + if (hdr->e_phnum < 1) + return false; + if (hdr->e_phentsize != sizeof (Elf64_Phdr)) + return false; + /* + * 3. Validate that this is an executable for our target architecture. + */ + if ((hdr->e_type != ET_EXEC) + && (hdr->e_type != ET_DYN)) /* DJW: PIE makes ET_DYN */ + return false; + if (hdr->e_machine != EM_TARGET) + return false; + + return true; +} + +/* + * Align (addr) down to (align) boundary. Returns 1 if (align) is not a + * non-zero power of 2. + */ +static int align_down(Elf64_Addr addr, Elf64_Xword align, + Elf64_Addr *out_result) +{ + if (align > 0 && (align & (align - 1)) == 0) { + *out_result = addr & -align; + return 0; + } + else + return 1; +} + +/* + * Align (addr) up to (align) boundary. Returns 1 if an overflow would occur or + * (align) is not a non-zero power of 2, otherwise result in (*out_result) and + * 0. + */ +static int align_up(Elf64_Addr addr, Elf64_Xword align, Elf64_Addr *out_result) +{ + Elf64_Addr result; + + if (align > 0 && (align & (align - 1)) == 0) { + if (check_add_overflow(addr, (align - 1), &result)) + return 1; + result = result & -align; + *out_result = result; + return 0; + } + else + return 1; +} + +static int elf_read(struct file *file, void *buf, size_t len, loff_t pos) +{ + ssize_t rv; + + rv = kernel_read(file, buf, len, &pos); + if (unlikely(rv != len)) { + return (rv < 0) ? rv : -EIO; + } + return 0; +} + +static int rex_parse_maps(union bpf_attr *attr, struct bpf_prog *prog, + u64 addr_start) +{ + u64 map_offs[MAX_USED_MAPS]; + struct bpf_map **used_maps; + int idx, ret = 0; + + if (attr->map_cnt >= MAX_USED_MAPS) + return -EINVAL; + + if (copy_from_bpfptr(map_offs, USER_BPFPTR((void *)(attr->map_offs)), + sizeof(u64) * attr->map_cnt) != 0) + return -EFAULT; + + used_maps = kmalloc(sizeof(*used_maps) * attr->map_cnt, GFP_KERNEL); + if (!used_maps) + return -ENOMEM; + + for (idx = 0; idx < attr->map_cnt; idx++) { + u64 *map_addr = (u64 *)(addr_start + map_offs[idx]); + struct bpf_map *curr = bpf_map_get(*map_addr); + unsigned int level; + pte_t *pte = lookup_address((unsigned long)map_addr, &level); + bool is_ro = !pte_write(*pte); + unsigned long start = (unsigned long)map_addr & PAGE_MASK; + unsigned long end = ((unsigned long)map_addr + sizeof(curr)) & + PAGE_MASK; + int nr_pages = start == end ? 1 : 2; + + if (IS_ERR(curr)) { + ret = PTR_ERR(curr); + goto free_used_maps; + } + + used_maps[idx] = curr; + + /* Maps might (or will always?) be in .data, which is read-only */ + if (is_ro) + set_memory_rw(start, nr_pages); + *map_addr = (u64)curr; + if (is_ro) + set_memory_ro(start, nr_pages); + } + prog->aux->used_maps = used_maps; + prog->aux->used_map_cnt = attr->map_cnt; + + return 0; + +free_used_maps: + kfree(used_maps); + return ret; +} + +static int rex_parse_relas(union bpf_attr *attr, u64 addr_start) +{ + int i = 0; + int ret = 0; + u64 relas_size = attr->nr_dyn_relas * sizeof(struct rex_rela_dyn); + struct rex_rela_dyn *relas = kmalloc_array(attr->nr_dyn_relas, + sizeof(*relas), GFP_KERNEL); + + if (!relas) + return -ENOMEM; + + if (copy_from_bpfptr(relas, USER_BPFPTR((void *)(attr->dyn_relas)), + relas_size) != 0) { + ret = -EFAULT; + goto free_relas; + } + + for (i = 0; i < attr->nr_dyn_relas; i++) { + u64 *abs_addr; + + if (ELF64_R_TYPE(relas[i].info) != R_X86_64_RELATIVE) { + ret = -EINVAL; + goto free_relas; + } + + abs_addr = (u64 *)(addr_start + relas[i].offset); + *abs_addr = addr_start + relas[i].addend; + } + +free_relas: + kfree(relas); + return ret; +} + +static int rex_parse_dyn_syms(union bpf_attr *attr, u64 addr_start, struct bpf_prog *prog) +{ + int i = 0, ret = 0; + u64 syms_size = attr->nr_dyn_syms * sizeof(struct rex_dyn_sym); + struct rex_dyn_sym *syms = kmalloc_array(attr->nr_dyn_syms, + sizeof(*syms), GFP_KERNEL); + char name[KSYM_NAME_LEN] = { 0 }; + + if (!syms) + return -ENOMEM; + + if (copy_from_bpfptr(syms, USER_BPFPTR((void *)attr->dyn_syms), + syms_size) != 0) { + ret = -EFAULT; + goto free_syms; + } + + for (i = 0; i < attr->nr_dyn_syms; i++) { + u64 *abs_addr = (u64 *)(addr_start + syms[i].offset); + u64 sym_addr; + + memset(name, 0, KSYM_NAME_LEN); + ret = strncpy_from_user(name, syms[i].symbol, KSYM_NAME_LEN); + if (ret == KSYM_NAME_LEN) + ret = -E2BIG; + if (ret < 0) + goto free_syms; + + sym_addr = kallsyms_lookup_name(name); + if (!sym_addr) { + ret = -EINVAL; + goto free_syms; + } + + /* A better way is to create a dedicated kprobe program type that can + * override return values */ + if (IS_ENABLED(CONFIG_BPF_KPROBE_OVERRIDE)) { + extern void just_return_func(void); + if (sym_addr == (u64)just_return_func) + prog->kprobe_override = 1; + } + + *abs_addr = sym_addr; + } + + ret = 0; + +free_syms: + kfree(syms); + return ret; +} + +static int rex_parse_text_syms(union bpf_attr *attr, u64 addr_start, + struct bpf_prog *prog) +{ + int ret = 0; + u64 syms_size = attr->nr_text_syms * sizeof(struct rex_text_sym); + char name[KSYM_NAME_LEN] = { 0 }; + struct rex_text_sym *text_syms = kmalloc_array( + attr->nr_text_syms, sizeof(*text_syms), GFP_KERNEL); + struct bpf_ksym *ksyms; + + if (!text_syms) + return -ENOMEM; + + ksyms = kmalloc_array(attr->nr_text_syms, sizeof(*ksyms), + GFP_KERNEL | __GFP_ZERO); + if (!ksyms) { + ret = -ENOMEM; + goto free_text_syms; + } + + if (copy_from_bpfptr(text_syms, USER_BPFPTR((void *)attr->text_syms), + syms_size) != 0) { + ret = -EFAULT; + goto free_ksyms; + } + + for (int i = 0; i < attr->nr_text_syms; i++) { + u64 abs_addr = addr_start + text_syms[i].offset; + char *sym = ksyms[i].name; + const char *end = sym + KSYM_NAME_LEN; + + memset(name, 0, KSYM_NAME_LEN); + ret = strncpy_from_user(name, text_syms[i].symbol, + KSYM_NAME_LEN); + if (ret == KSYM_NAME_LEN) + ret = -E2BIG; + if (ret < 0) + goto free_ksyms; + + ksyms[i].prog = true; + ksyms[i].start = abs_addr; + ksyms[i].end = abs_addr + text_syms[i].size; + + sym += snprintf(sym, KSYM_NAME_LEN, "rex_prog_"); + sym = bin2hex(sym, prog->tag, sizeof(prog->tag)); + snprintf(sym, (size_t)(end - sym), "::%s", name); + + INIT_LIST_HEAD(&ksyms[i].lnode); + } + + prog->aux->rex_syms = ksyms; + prog->aux->nr_syms = attr->nr_text_syms; + ret = 0; + + /* Don't free ksyms on success as we have already given away ownership */ + goto free_text_syms; + +free_ksyms: + kfree(ksyms); +free_text_syms: + kfree(text_syms); + return ret; +} + +#define MAX_PROG_SZ (8192 << 4) +static int bpf_prog_load_rex_base(union bpf_attr *attr, bpfptr_t uattr) +{ + enum bpf_prog_type type = attr->prog_type; + struct bpf_prog *prog, *dst_prog = NULL; + struct btf *attach_btf = NULL; + int err; + char license[128]; + bool is_gpl; + + void *mem; + Elf64_Phdr *phdr = NULL; + Elf64_Ehdr *ehdr = NULL; + Elf64_Addr e_entry; /* Program entry point */ + Elf64_Addr e_end; /* Highest memory address occupied */ + struct file *filp; + size_t ph_size; + Elf64_Addr plast_vaddr = 0; + Elf64_Half ph_i; + u64 addr_start = 0; + int *vm_size = NULL, *sec_off = NULL; + int total_vm = 0; + + if (CHECK_ATTR(BPF_PROG_LOAD)) + return -EINVAL; + if (attr->prog_flags & ~(BPF_F_STRICT_ALIGNMENT | + BPF_F_ANY_ALIGNMENT | + BPF_F_TEST_STATE_FREQ | + BPF_F_SLEEPABLE | + BPF_F_TEST_RND_HI32)) + return -EINVAL; if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && (attr->prog_flags & BPF_F_ANY_ALIGNMENT) && - !bpf_cap) - goto put_token; + !bpf_capable()) + return -EPERM; - /* Intent here is for unprivileged_bpf_disabled to block BPF program - * creation for unprivileged users; other actions depend - * on fd availability and access to bpffs, so are dependent on - * object creation success. Even with unprivileged BPF disabled, - * capability checks are still carried out for these - * and other operations. - */ - if (sysctl_unprivileged_bpf_disabled && !bpf_cap) - goto put_token; + /* copy eBPF program license from user space */ + if (strncpy_from_bpfptr(license, + make_bpfptr(attr->license, uattr.is_kernel), + sizeof(license) - 1) < 0) + return -EFAULT; + license[sizeof(license) - 1] = 0; - if (attr->insn_cnt == 0 || - attr->insn_cnt > (bpf_cap ? BPF_COMPLEXITY_LIMIT_INSNS : BPF_MAXINSNS)) { - err = -E2BIG; - goto put_token; - } - if (type != BPF_PROG_TYPE_SOCKET_FILTER && - type != BPF_PROG_TYPE_CGROUP_SKB && - !bpf_cap) - goto put_token; + /* eBPF programs must be GPL compatible to use GPL-ed functions */ + is_gpl = license_is_gpl_compatible(license); - if (is_net_admin_prog_type(type) && !bpf_token_capable(token, CAP_NET_ADMIN)) - goto put_token; - if (is_perfmon_prog_type(type) && !bpf_token_capable(token, CAP_PERFMON)) - goto put_token; + /* Root-only for now */ + if (!bpf_capable()) + return -EPERM; + + if (is_net_admin_prog_type(type) && !capable(CAP_NET_ADMIN) && !capable(CAP_SYS_ADMIN)) + return -EPERM; + if (is_perfmon_prog_type(type) && !perfmon_capable()) + return -EPERM; + + /* Userspace should always supply symbol table */ + if (!attr->nr_text_syms) + return -EINVAL; /* attach_prog_fd/attach_btf_obj_fd can specify fd of either bpf_prog * or btf, we need to check which one it is @@ -2951,33 +4522,27 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) if (IS_ERR(dst_prog)) { dst_prog = NULL; attach_btf = btf_get_by_fd(attr->attach_btf_obj_fd); - if (IS_ERR(attach_btf)) { - err = -EINVAL; - goto put_token; - } + if (IS_ERR(attach_btf)) + return -EINVAL; if (!btf_is_kernel(attach_btf)) { /* attaching through specifying bpf_prog's BTF * objects directly might be supported eventually */ btf_put(attach_btf); - err = -ENOTSUPP; - goto put_token; + return -ENOTSUPP; } } } else if (attr->attach_btf_id) { /* fall back to vmlinux BTF, if BTF type ID is specified */ attach_btf = bpf_get_btf_vmlinux(); - if (IS_ERR(attach_btf)) { - err = PTR_ERR(attach_btf); - goto put_token; - } - if (!attach_btf) { - err = -EINVAL; - goto put_token; - } + if (IS_ERR(attach_btf)) + return PTR_ERR(attach_btf); + if (!attach_btf) + return -EINVAL; btf_get(attach_btf); } + bpf_prog_load_fixup_attach_type(attr); if (bpf_prog_load_check_attach(type, attr->expected_attach_type, attach_btf, attr->attach_btf_id, dst_prog)) { @@ -2985,8 +4550,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - err = -EINVAL; - goto put_token; + return -EINVAL; } /* plain bpf_prog allocation */ @@ -2996,8 +4560,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) bpf_prog_put(dst_prog); if (attach_btf) btf_put(attach_btf); - err = -EINVAL; - goto put_token; + return -ENOMEM; } prog->expected_attach_type = attr->expected_attach_type; @@ -3005,97 +4568,261 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) prog->aux->attach_btf = attach_btf; prog->aux->attach_btf_id = attr->attach_btf_id; prog->aux->dst_prog = dst_prog; - prog->aux->dev_bound = !!attr->prog_ifindex; - prog->aux->xdp_has_frags = attr->prog_flags & BPF_F_XDP_HAS_FRAGS; - - /* move token into prog->aux, reuse taken refcnt */ - prog->aux->token = token; - token = NULL; + prog->aux->offload_requested = !!attr->prog_ifindex; prog->aux->user = get_current_user(); prog->len = attr->insn_cnt; - err = -EFAULT; - if (copy_from_bpfptr(prog->insns, - make_bpfptr(attr->insns, uattr.is_kernel), - bpf_prog_insn_size(prog)) != 0) - goto free_prog; - /* copy eBPF program license from user space */ - if (strncpy_from_bpfptr(license, - make_bpfptr(attr->license, uattr.is_kernel), - sizeof(license) - 1) < 0) - goto free_prog; - license[sizeof(license) - 1] = 0; - - /* eBPF programs must be GPL compatible to use GPL-ed functions */ - prog->gpl_compatible = license_is_gpl_compatible(license) ? 1 : 0; - - if (attr->signature) { - err = bpf_prog_verify_signature(prog, attr, uattr.is_kernel); - if (err) - goto free_prog; - } prog->orig_prog = NULL; - prog->jited = 0; + prog->jited = 1; /* DJW: we are always 'jited' */ + prog->no_bpf = 1; atomic64_set(&prog->aux->refcnt, 1); - + prog->gpl_compatible = is_gpl ? 1 : 0; if (bpf_prog_is_dev_bound(prog->aux)) { err = bpf_prog_dev_bound_init(prog, attr); if (err) - goto free_prog; - } - - if (type == BPF_PROG_TYPE_EXT && dst_prog && - bpf_prog_is_dev_bound(dst_prog->aux)) { - err = bpf_prog_dev_bound_inherit(prog, dst_prog); - if (err) - goto free_prog; - } - - /* - * Bookkeeping for managing the program attachment chain. - * - * It might be tempting to set attach_tracing_prog flag at the attachment - * time, but this will not prevent from loading bunch of tracing prog - * first, then attach them one to another. - * - * The flag attach_tracing_prog is set for the whole program lifecycle, and - * doesn't have to be cleared in bpf_tracing_link_release, since tracing - * programs cannot change attachment target. - */ - if (type == BPF_PROG_TYPE_TRACING && dst_prog && - dst_prog->type == BPF_PROG_TYPE_TRACING) { - prog->aux->attach_tracing_prog = true; + goto free_prog_sec; } /* find program type: socket_filter vs tracing_filter */ err = find_prog_type(type, prog); if (err < 0) - goto free_prog; + goto free_prog_sec; prog->aux->load_time = ktime_get_boottime_ns(); err = bpf_obj_name_cpy(prog->aux->name, attr->prog_name, sizeof(attr->prog_name)); if (err < 0) - goto free_prog; + goto free_prog_sec; - err = security_bpf_prog_load(prog, attr, token, uattr.is_kernel); - if (err) + bpf_get_trace_printk_proto(); + + filp = fget(attr->rustfd); + ehdr = kmalloc(sizeof(Elf64_Ehdr), GFP_KERNEL); + if (ehdr == NULL) { + fput(filp); + err = -ENOMEM; goto free_prog_sec; + } - /* run eBPF verifier */ - err = bpf_check(&prog, attr, uattr, uattr_size); - if (err < 0) - goto free_used_maps; + err = elf_read(filp, ehdr, sizeof(Elf64_Ehdr), 0); + if (err) + goto error_ehdr; - prog = bpf_prog_select_runtime(prog, &err); - if (err < 0) - goto free_used_maps; + if (!ehdr_is_valid(ehdr)) { + err = -EINVAL; + goto error_ehdr; + } - err = bpf_prog_mark_insn_arrays_ready(prog); - if (err < 0) + e_entry = ehdr->e_entry; + ph_size = ehdr->e_phnum * ehdr->e_phentsize; + phdr = kmalloc(ph_size, GFP_KERNEL); + if (!phdr) { + err = -ENOMEM; + goto error_ehdr; + } + + vm_size = kmalloc(sizeof(int) * ehdr->e_phnum, GFP_KERNEL); + sec_off = kmalloc(sizeof(int) * ehdr->e_phnum, GFP_KERNEL); + if ((!vm_size) || (!sec_off)) { + err = -ENOMEM; + goto error_ehdr; + } + + err = elf_read(filp, phdr, ph_size, ehdr->e_phoff); + if (err) + goto error_phdr; + + /* + * Load all program segments with the PT_LOAD directive. + */ + e_end = 0; + err = -EINVAL; + for (ph_i = 0; ph_i < ehdr->e_phnum; ph_i++) { + Elf64_Addr p_vaddr = phdr[ph_i].p_vaddr; + Elf64_Xword p_filesz = phdr[ph_i].p_filesz; + Elf64_Xword p_memsz = phdr[ph_i].p_memsz; + Elf64_Xword p_align = phdr[ph_i].p_align; + Elf64_Addr temp, p_vaddr_start, p_vaddr_end; + + if (phdr[ph_i].p_type != PT_LOAD){ + vm_size[ph_i] = 0; + sec_off[ph_i] = 0; + continue; + } + + /* + * The ELF specification mandates that program headers are sorted on + * p_vaddr in ascending order. Enforce this, at the same time avoiding + * any surprises later. + */ + if (p_vaddr < plast_vaddr) + goto error_phdr; + else + plast_vaddr = p_vaddr; + + /* + * Compute p_vaddr_start = p_vaddr, aligned down to requested alignment + * and verify result is within range. + */ + if (align_down(p_vaddr, p_align, &p_vaddr_start)) + goto error_phdr; + + /* + * Disallow overlapping segments. This may be overkill, but in practice + * the Solo5 toolchains do not produce such executables. + */ + if (p_vaddr_start < e_end) + goto error_phdr; + + /* + * Verify p_vaddr + p_filesz is within range. + */ + if (p_vaddr >= MAX_PROG_SZ) + goto error_phdr; + if (check_add_overflow(p_vaddr, p_filesz, &temp)) + goto error_phdr; + if (temp > MAX_PROG_SZ) + goto error_phdr; + + /* + * Compute p_vaddr_end = p_vaddr + p_memsz, aligned up to requested + * alignment and verify result is within range. + */ + if (p_memsz < p_filesz) + goto error_phdr; + if (check_add_overflow(p_vaddr, p_memsz, &p_vaddr_end)) + goto error_phdr; + if (align_up(p_vaddr_end, p_align, &p_vaddr_end)) + goto error_phdr; + if (p_vaddr_end > MAX_PROG_SZ) + goto error_phdr; + + /* Enforce 4k alignment for now */ + if (p_align != 1UL << PAGE_SHIFT) + goto error_phdr; + + /* + * Keep track of the highest byte of memory occupied by the program. + */ + if (p_vaddr_end > e_end) { + e_end = p_vaddr_end; + } + + /* + * Memory protection flags should be applied to the aligned address + * range (p_vaddr_start .. p_vaddr_end). Before we apply them, also + * verify that the address range is aligned to the architectural page + * size. + */ + if (p_vaddr_start & (EM_PAGE_SIZE - 1)) + goto error_phdr; + if (p_vaddr_end & (EM_PAGE_SIZE - 1)) + goto error_phdr; + + vm_size[ph_i] = round_up(p_vaddr_end - p_vaddr_start, p_align); + sec_off[ph_i] = p_vaddr - p_vaddr_start; + } + + /* Allocate enough space to hold the largest address */ + total_vm = e_end; + + mem = __vmalloc(total_vm, GFP_KERNEL_ACCOUNT | __GFP_ZERO | GFP_USER); + if (!mem) { + err = -ENOMEM; + goto error_phdr; + } + prog->mem.mem = mem; + addr_start = (u64)mem; + + prog->mem.total_page = total_vm >> PAGE_SHIFT; + + for (ph_i = 0; ph_i < ehdr->e_phnum; ph_i++) { + Elf64_Xword p_filesz = phdr[ph_i].p_filesz; + + int prot; + void *readbuf; + int page_cnt = (vm_size[ph_i] >> PAGE_SHIFT); + u64 map_addr = (u64)mem + phdr[ph_i].p_vaddr - sec_off[ph_i]; + + if (phdr[ph_i].p_type != PT_LOAD) + continue; + + prot = PROT_NONE; + if (phdr[ph_i].p_flags & PF_R) + prot |= PROT_READ; + if (phdr[ph_i].p_flags & PF_W) + prot |= PROT_WRITE; + if (phdr[ph_i].p_flags & PF_X) + prot |= PROT_EXEC; + if ((prot & PROT_WRITE) && (prot & PROT_EXEC)) + goto error_vm; + + readbuf = kmalloc(p_filesz, GFP_KERNEL); + if (!readbuf) { + err = -ENOMEM; + goto error_vm; + } + err = elf_read(filp, readbuf, p_filesz, phdr[ph_i].p_offset); + if (err) { + kfree(readbuf); + goto error_vm; + } + + memcpy(mem + phdr[ph_i].p_vaddr, readbuf, p_filesz); + + // Set correct permission + if ((prot & PROT_READ) && (prot & PROT_EXEC)) { + set_memory_ro(map_addr, page_cnt); + set_memory_x(map_addr, page_cnt); + } else if ((prot & PROT_READ) && (prot & PROT_WRITE)) { + set_memory_rw(map_addr, page_cnt); // acutally not needed + } else if (prot & PROT_READ) { + set_memory_ro(map_addr, page_cnt); + } else { + kfree(readbuf); + set_memory_nx((u64)mem, prog->mem.total_page); + set_memory_rw((u64)mem, prog->mem.total_page); + err = -EINVAL; + goto error_vm; + } + kfree(readbuf); + } + + kfree(ehdr); + kfree(phdr); + kfree(vm_size); + kfree(sec_off); + fput(filp); + + prog->bpf_func = __rex_prog_empty; + prog->jited_len = 0; + + BUILD_BUG_ON(sizeof(prog->tag) != sizeof(unsigned long)); + ptr_to_hashval((const void *)addr_start, (unsigned long *)&prog->tag); + + if (attr->map_cnt) { + err = rex_parse_maps(attr, prog, addr_start); + if (err) + goto free_used_maps; + } + + if (attr->nr_dyn_relas) { + err = rex_parse_relas(attr, addr_start); + if (err) + goto free_used_maps; + } + + if (attr->nr_dyn_syms) { + err = rex_parse_dyn_syms(attr, addr_start, prog); + if (err) + goto free_used_maps; + } + + err = rex_parse_text_syms(attr, addr_start, prog); + if (err) goto free_used_maps; err = bpf_prog_alloc_id(prog); @@ -3116,7 +4843,7 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) * Also, any failure handling from this point onwards must * be using bpf_prog_put() given the program is exposed. */ - bpf_prog_kallsyms_add(prog); + rex_prog_kallsyms_add(prog); perf_event_bpf_event(prog, PERF_BPF_EVENT_PROG_LOAD, 0); bpf_audit_prog(prog, BPF_AUDIT_LOAD); @@ -3130,21 +4857,100 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size) * period before we can tear down JIT memory since symbols * are already exposed under kallsyms. */ - __bpf_prog_put_noref(prog, prog->aux->real_func_cnt); + __bpf_prog_put_noref(prog, prog->aux->func_cnt); return err; - +error_vm: + vfree(mem); +error_phdr: + kfree(phdr); + kfree(vm_size); + kfree(sec_off); +error_ehdr: + kfree(ehdr); + fput(filp); free_prog_sec: - security_bpf_prog_free(prog); -free_prog: free_uid(prog->aux->user); + security_bpf_prog_free(prog); +// free_prog: if (prog->aux->attach_btf) btf_put(prog->aux->attach_btf); bpf_prog_free(prog); -put_token: - bpf_token_put(token); return err; } +extern int scx_enable_rex(struct bpf_prog *base, + struct rex_sched_ops_sym __user *usyms, u32 nr_syms, + u64 ops_flags, u32 timeout_ms, u32 exit_dump_len, + const char *user_name); +extern int scx_disable_rex(void); + +static int bpf_sched_ext_attach_rex(union bpf_attr *attr, bpfptr_t uattr) +{ + struct bpf_prog *base; + int err; + + pr_info("bpf_syscall: BPF_SCHED_EXT_ATTACH_REX called (fd=%u, nr_syms=%u)\n", + attr->sched_ext_attach.base_prog_fd, + attr->sched_ext_attach.nr_sched_ops_syms); + + if (!bpf_capable()) + return -EPERM; + + if (!attr->sched_ext_attach.base_prog_fd || + !attr->sched_ext_attach.sched_ops_syms || + !attr->sched_ext_attach.nr_sched_ops_syms) + return -EINVAL; + + base = bpf_prog_get(attr->sched_ext_attach.base_prog_fd); + if (IS_ERR(base)) + return PTR_ERR(base); + + if (base->type != BPF_PROG_TYPE_REX_BASE) { + err = -EINVAL; + goto put_prog; + } + + pr_info("bpf_syscall: Rex base prog verified, forwarding to scx_enable_rex()\n"); + + /* Ensure the user-provided name is NUL-terminated before handing it + * to scx_enable_rex(). The UAPI struct is a char[128]; treat an empty + * first byte as "no name provided, fall back to base->aux->name". */ + attr->sched_ext_attach.name[sizeof(attr->sched_ext_attach.name) - 1] = '\0'; + + err = scx_enable_rex(base, + u64_to_user_ptr(attr->sched_ext_attach.sched_ops_syms), + attr->sched_ext_attach.nr_sched_ops_syms, + attr->sched_ext_attach.ops_flags, + attr->sched_ext_attach.timeout_ms, + attr->sched_ext_attach.exit_dump_len, + attr->sched_ext_attach.name); + if (err) + goto put_prog; + + pr_info("bpf_syscall: BPF_SCHED_EXT_ATTACH_REX succeeded\n"); + + /* + * Keep the bpf_prog_get() reference: the scheduler callbacks point + * into base->mem.mem which must stay alive. scx_enable_rex() saved + * the pointer; scx_disable_rex() will release it on detach. + */ + return 0; + +put_prog: + bpf_prog_put(base); + return err; +} + +static int bpf_sched_ext_detach_rex(void) +{ + pr_info("bpf_syscall: BPF_SCHED_EXT_DETACH_REX called\n"); + + if (!bpf_capable()) + return -EPERM; + + return scx_disable_rex(); +} + #define BPF_OBJ_LAST_FIELD path_fd static int bpf_obj_pin(const union bpf_attr *attr) @@ -3200,6 +5006,7 @@ void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type, link->ops = ops; link->prog = prog; link->attach_type = attach_type; + prog->saved_state->link = link; } void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, @@ -6248,6 +8055,18 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size) case BPF_PROG_LOAD: err = bpf_prog_load(&attr, uattr, size); break; + case BPF_PROG_LOAD_REX_BASE: + err = bpf_prog_load_rex_base(&attr, uattr); + break; + case BPF_PROG_LOAD_REX: + err = bpf_prog_load_rex(&attr, uattr); + break; + case BPF_SCHED_EXT_ATTACH_REX: + err = bpf_sched_ext_attach_rex(&attr, uattr); + break; + case BPF_SCHED_EXT_DETACH_REX: + err = bpf_sched_ext_detach_rex(); + break; case BPF_OBJ_PIN: err = bpf_obj_pin(&attr); break; @@ -6343,6 +8162,26 @@ static int __sys_bpf(enum bpf_cmd cmd, bpfptr_t uattr, unsigned int size) case BPF_TOKEN_CREATE: err = token_create(&attr); break; + case BPF_PROG_TERMINATE: + /* printk("Starting terminate syscall prog_id : %d\n", */ + /* attr.prog_id); */ + /* struct bpf_prog *prog; */ + /* int cpu_id; */ + /* prog = bpf_prog_by_id(attr.prog_id); */ + /* if (IS_ERR(prog) || (cpu_id = prog->saved_state->cpu_id) < 0) { */ + /* printk("bpf prog_id : %d not found or not running!" */ + /* "Not executing terminate.\n", */ + /* attr.prog_id); */ + /* err = -EINVAL; */ + /**/ + /* } else { */ + /* printk("Sending rex_terminate IPI to CPU : %d\n", */ + /* cpu_id); */ + /* smp_call_function_single(cpu_id, rex_terminate, */ + /* (void *)prog, 1); */ + /* err = 0; */ + /* } */ + break; case BPF_PROG_STREAM_READ_BY_FD: err = prog_stream_read(&attr); break; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 496dff740dcafe..d2b4d330a37319 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -2529,13 +2529,7 @@ static int migration_cpu_stop(void *data) * __migrate_task() such that we will not miss enforcing cpus_ptr * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. */ - flush_smp_call_function_queue(); - - /* - * We may change the underlying rq, but the locks held will - * appropriately be "transferred" when switching. - */ - context_unsafe_alias(rq); + flush_smp_call_function_queue(task_pt_regs(current)); raw_spin_lock(&p->pi_lock); rq_lock(rq, &rf); diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c index 064eaa76be4b9f..9c21aaa175fec1 100644 --- a/kernel/sched/ext.c +++ b/kernel/sched/ext.c @@ -2831,9 +2831,13 @@ static void scx_watchdog_workfn(struct work_struct *work) WRITE_ONCE(scx_watchdog_timestamp, jiffies); + pr_info_ratelimited("sched_ext: watchdog tick (checking all CPUs)\n"); + for_each_online_cpu(cpu) { - if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) + if (unlikely(check_rq_for_timeouts(cpu_rq(cpu)))) { + pr_warn("sched_ext: watchdog detected timeout on CPU %d!\n", cpu); break; + } cond_resched(); } @@ -4355,6 +4359,9 @@ static void free_kick_syncs(void) } } +static DEFINE_MUTEX(scx_rex_mutex); +static struct bpf_prog *scx_rex_base_prog; + static void scx_disable_workfn(struct kthread_work *work) { struct scx_sched *sch = container_of(work, struct scx_sched, disable_work); @@ -4498,6 +4505,26 @@ static void scx_disable_workfn(struct kthread_work *work) mutex_unlock(&scx_enable_mutex); + /* + * Drop refs taken when this sched was attached via the Rex syscall + * path. The struct_ops path drops its equivalents via bpf_scx_unreg(); + * Rex has no link, so we do it here so cleanup fires for every disable + * kind (UNREG, ERROR_STALL, watchdog, sysrq, ...). + */ + if (sch->rex_base) { + struct bpf_prog *base = sch->rex_base; + + sch->rex_base = NULL; + + mutex_lock(&scx_rex_mutex); + if (scx_rex_base_prog == base) + scx_rex_base_prog = NULL; + mutex_unlock(&scx_rex_mutex); + + kobject_put(&sch->kobj); + bpf_prog_put(base); + } + WARN_ON_ONCE(scx_set_enable_state(SCX_DISABLED) != SCX_DISABLING); done: scx_bypass(false); @@ -5100,6 +5127,8 @@ static void scx_enable_workfn(struct kthread_work *work) if (WARN_ON_ONCE(READ_ONCE(scx_aborting))) WRITE_ONCE(scx_aborting, false); + pr_info("sched_ext: [1/6] state -> SCX_ENABLING\n"); + atomic_long_set(&scx_nr_rejected, 0); for_each_possible_cpu(cpu) @@ -5120,6 +5149,7 @@ static void scx_enable_workfn(struct kthread_work *work) scx_idle_enable(ops); if (sch->ops.init) { + pr_info("sched_ext: [2/6] calling ops.init() ...\n"); ret = SCX_CALL_OP_RET(sch, SCX_KF_UNLOCKED, init, NULL); if (ret) { ret = ops_sanitize_err(sch, "init", ret); @@ -5128,6 +5158,7 @@ static void scx_enable_workfn(struct kthread_work *work) goto err_disable; } sch->exit_info->flags |= SCX_EFLAG_INITIALIZED; + pr_info("sched_ext: [2/6] ops.init() returned successfully\n"); } for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++) @@ -5165,7 +5196,9 @@ static void scx_enable_workfn(struct kthread_work *work) WRITE_ONCE(scx_watchdog_timeout, timeout); WRITE_ONCE(scx_watchdog_timestamp, jiffies); queue_delayed_work(system_unbound_wq, &scx_watchdog_work, - READ_ONCE(scx_watchdog_timeout) / 2); + scx_watchdog_timeout / 2); + pr_info("sched_ext: [3/6] watchdog armed (timeout=%lu ms)\n", + jiffies_to_msecs(timeout)); /* * Once __scx_enabled is set, %current can be switched to SCX anytime. @@ -5191,6 +5224,7 @@ static void scx_enable_workfn(struct kthread_work *work) WARN_ON_ONCE(scx_init_task_enabled); scx_init_task_enabled = true; + pr_info("sched_ext: [4/6] initializing all existing tasks for SCX ...\n"); /* * Enable ops for every task. Fork is excluded by scx_fork_rwsem @@ -5245,6 +5279,7 @@ static void scx_enable_workfn(struct kthread_work *work) */ WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL)); static_branch_enable(&__scx_enabled); + pr_info("sched_ext: [5/6] scx_enabled=true, switching all tasks to SCX class ...\n"); /* * We're fully committed and can't fail. The task READY -> ENABLED @@ -5283,6 +5318,7 @@ static void scx_enable_workfn(struct kthread_work *work) if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL)) static_branch_enable(&__scx_switched_all); + pr_info("sched_ext: [6/6] state -> SCX_ENABLED. Scheduler takeover COMPLETE!\n"); pr_info("sched_ext: BPF scheduler \"%s\" enabled%s\n", sch->ops.name, scx_switched_all() ? "" : " (partial)"); kobject_uevent(&sch->kobj, KOBJ_ADD); @@ -5485,6 +5521,192 @@ static int bpf_scx_reg(void *kdata, struct bpf_link *link) return scx_enable(kdata, link); } +int scx_enable_rex(struct bpf_prog *base, + struct rex_sched_ops_sym __user *usyms, u32 nr_syms, + u64 ops_flags, u32 timeout_ms, u32 exit_dump_len, + const char *user_name) +{ + struct sched_ext_ops *ops; + struct rex_sched_ops_sym *syms; + struct scx_sched *sch; + char name_buf[128]; + u32 i; + int err; + + pr_info("sched_ext_rex: === REX ENABLE START === nr_syms=%u\n", nr_syms); + + if (nr_syms > 64) + return -EINVAL; + + syms = kvmalloc_array(nr_syms, sizeof(*syms), GFP_KERNEL); + if (!syms) + return -ENOMEM; + + if (copy_from_user(syms, usyms, nr_syms * sizeof(*syms))) { + err = -EFAULT; + goto free_syms; + } + + ops = kzalloc(sizeof(*ops), GFP_KERNEL); + if (!ops) { + err = -ENOMEM; + goto free_syms; + } + + for (i = 0; i < nr_syms; i++) { + void *fn; + long name_len; + bool matched; + + name_len = strncpy_from_user(name_buf, syms[i].name, + sizeof(name_buf)); + if (name_len <= 0 || name_len >= sizeof(name_buf)) { + err = -EFAULT; + goto free_ops; + } + + if (syms[i].offset >= (u64)base->mem.total_page << PAGE_SHIFT) { + err = -EINVAL; + goto free_ops; + } + + fn = (void *)((u64)base->mem.mem + syms[i].offset); + matched = false; + +#define SCX_OP_MATCH(field) \ + do { if (!strcmp(name_buf, #field)) { ops->field = fn; matched = true; } } while (0) + + if (!matched) SCX_OP_MATCH(select_cpu); + if (!matched) SCX_OP_MATCH(enqueue); + if (!matched) SCX_OP_MATCH(dequeue); + if (!matched) SCX_OP_MATCH(dispatch); + if (!matched) SCX_OP_MATCH(tick); + if (!matched) SCX_OP_MATCH(runnable); + if (!matched) SCX_OP_MATCH(running); + if (!matched) SCX_OP_MATCH(stopping); + if (!matched) SCX_OP_MATCH(quiescent); + if (!matched) SCX_OP_MATCH(yield); + if (!matched) SCX_OP_MATCH(core_sched_before); + if (!matched) SCX_OP_MATCH(set_weight); + if (!matched) SCX_OP_MATCH(set_cpumask); + if (!matched) SCX_OP_MATCH(update_idle); + if (!matched) SCX_OP_MATCH(cpu_acquire); + if (!matched) SCX_OP_MATCH(cpu_release); + if (!matched) SCX_OP_MATCH(init_task); + if (!matched) SCX_OP_MATCH(exit_task); + if (!matched) SCX_OP_MATCH(enable); + if (!matched) SCX_OP_MATCH(disable); + if (!matched) SCX_OP_MATCH(dump); + if (!matched) SCX_OP_MATCH(dump_cpu); + if (!matched) SCX_OP_MATCH(dump_task); +#ifdef CONFIG_EXT_GROUP_SCHED + if (!matched) SCX_OP_MATCH(cgroup_init); + if (!matched) SCX_OP_MATCH(cgroup_exit); + if (!matched) SCX_OP_MATCH(cgroup_prep_move); + if (!matched) SCX_OP_MATCH(cgroup_move); + if (!matched) SCX_OP_MATCH(cgroup_cancel_move); + if (!matched) SCX_OP_MATCH(cgroup_set_weight); + if (!matched) SCX_OP_MATCH(cgroup_set_bandwidth); + if (!matched) SCX_OP_MATCH(cgroup_set_idle); +#endif + if (!matched) SCX_OP_MATCH(cpu_online); + if (!matched) SCX_OP_MATCH(cpu_offline); + if (!matched) SCX_OP_MATCH(init); + if (!matched) SCX_OP_MATCH(exit); + +#undef SCX_OP_MATCH + + if (!matched) { + pr_err("sched_ext_rex: unknown callback \"%s\"\n", + name_buf); + err = -EINVAL; + goto free_ops; + } + pr_info("sched_ext_rex: matched callback \"%s\" at offset 0x%llx\n", + name_buf, syms[i].offset); + } + + ops->flags = ops_flags; + ops->timeout_ms = timeout_ms; + ops->exit_dump_len = exit_dump_len; + + /* + * Prefer the user-supplied scheduler name (from SchedExtOps::name in + * the Rust program's .struct_ops). Fall back to the base program's + * name when the caller didn't provide one (empty string) -- this + * preserves the pre-fix behaviour for older loaders. + */ + if (user_name && user_name[0] != '\0') + strscpy(ops->name, user_name, sizeof(ops->name)); + else + strscpy(ops->name, base->aux->name, sizeof(ops->name)); + pr_info("sched_ext_rex: all %u callbacks matched, calling scx_enable(\"%s\")\n", + nr_syms, ops->name); + + mutex_lock(&scx_rex_mutex); + + err = scx_enable(ops, NULL); + if (err) { + pr_err("sched_ext_rex: scx_enable() FAILED err=%d\n", err); + mutex_unlock(&scx_rex_mutex); + goto free_ops; + } + + /* + * Hand the bpf_prog_get() reference taken in bpf_sched_ext_attach_rex() + * over to the scx_sched. scx_disable_workfn() will release it for any + * disable kind, so we don't depend on userspace detach running. + */ + rcu_read_lock(); + sch = rcu_dereference(scx_root); + rcu_read_unlock(); + if (sch) + sch->rex_base = base; + scx_rex_base_prog = base; + mutex_unlock(&scx_rex_mutex); + + pr_info("sched_ext_rex: === REX ENABLE COMPLETE === scheduler is ACTIVE\n"); + kvfree(syms); + kfree(ops); + return 0; + +free_ops: + kfree(ops); +free_syms: + kvfree(syms); + return err; +} +EXPORT_SYMBOL_GPL(scx_enable_rex); + +int scx_disable_rex(void) +{ + struct scx_sched *sch; + + pr_info("sched_ext_rex: === REX DISABLE START ===\n"); + + mutex_lock(&scx_rex_mutex); + if (!scx_rex_base_prog) { + /* Already torn down by another path (e.g. watchdog). */ + mutex_unlock(&scx_rex_mutex); + pr_info("sched_ext_rex: === REX DISABLE: already disabled ===\n"); + return 0; + } + rcu_read_lock(); + sch = rcu_dereference(scx_root); + rcu_read_unlock(); + mutex_unlock(&scx_rex_mutex); + + if (sch) { + pr_info("sched_ext_rex: disabling scheduler, switching tasks back to CFS ...\n"); + scx_disable(SCX_EXIT_UNREG); + kthread_flush_work(&sch->disable_work); + } + + pr_info("sched_ext_rex: === REX DISABLE COMPLETE === back to default scheduler\n"); + return 0; +} +EXPORT_SYMBOL_GPL(scx_disable_rex); + static void bpf_scx_unreg(void *kdata, struct bpf_link *link) { struct sched_ext_ops *ops = kdata; diff --git a/kernel/sched/ext_internal.h b/kernel/sched/ext_internal.h index 00b450597f3e06..45c6f2d816c16e 100644 --- a/kernel/sched/ext_internal.h +++ b/kernel/sched/ext_internal.h @@ -916,6 +916,15 @@ struct scx_sched { struct irq_work error_irq_work; struct kthread_work disable_work; struct rcu_work rcu_work; + + /* + * Set when this scheduler was attached via BPF_SCHED_EXT_ATTACH_REX. + * Owns one bpf_prog_get() reference taken at attach time and the + * matching kobject_init_and_add() refcount; both are released by + * scx_disable_workfn() so cleanup happens for every disable kind + * (UNREG, ERROR_STALL, watchdog, sysrq, ...). + */ + struct bpf_prog *rex_base; }; enum scx_wake_flags { diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index a83be0c834ddb5..c80cbe83b5a069 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -377,7 +377,7 @@ static void do_idle(void) * RCU relies on this call to be done outside of an RCU read-side * critical section. */ - flush_smp_call_function_queue(); + flush_smp_call_function_queue(task_pt_regs(current)); schedule_idle(); if (unlikely(klp_patch_pending(current))) diff --git a/kernel/sched/smp.h b/kernel/sched/smp.h index 7f151d96dba966..cd495d57a05784 100644 --- a/kernel/sched/smp.h +++ b/kernel/sched/smp.h @@ -14,9 +14,9 @@ extern void sched_ttwu_pending(void *arg); extern bool call_function_single_prep_ipi(int cpu); #ifdef CONFIG_SMP -extern void flush_smp_call_function_queue(void); +extern void flush_smp_call_function_queue(struct pt_regs *regs); #else -static inline void flush_smp_call_function_queue(void) { } +static inline void flush_smp_call_function_queue(struct pt_regs *regs) { } #endif #endif /* _KERNEL_SCHED_SMP_H */ diff --git a/kernel/smp.c b/kernel/smp.c index f349960f79cad9..722a3cb1dc222a 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -35,6 +35,8 @@ #include "smpboot.h" #include "sched/smp.h" +#include + #define CSD_TYPE(_csd) ((_csd)->node.u_flags & CSD_FLAG_TYPE_MASK) struct call_function_data { @@ -49,7 +51,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct llist_head, call_single_queue); static DEFINE_PER_CPU(atomic_t, trigger_backtrace) = ATOMIC_INIT(1); -static void __flush_smp_call_function_queue(bool warn_cpu_offline); +static void __flush_smp_call_function_queue(struct pt_regs *regs, + bool warn_cpu_offline); int smpcfd_prepare_cpu(unsigned int cpu) { @@ -96,7 +99,7 @@ int smpcfd_dying_cpu(unsigned int cpu) * This runs with interrupts disabled inside the stopper task invoked by * stop_machine(), ensuring mutually exclusive CPU offlining and IPI flush. */ - __flush_smp_call_function_queue(false); + __flush_smp_call_function_queue(task_pt_regs(current), false); irq_work_run(); return 0; } @@ -458,9 +461,9 @@ static int generic_exec_single(int cpu, call_single_data_t *csd) * Invoked by arch to handle an IPI for call function single. * Must be called with interrupts disabled. */ -void generic_smp_call_function_single_interrupt(void) +void generic_smp_call_function_single_interrupt(struct pt_regs *regs) { - __flush_smp_call_function_queue(true); + __flush_smp_call_function_queue(regs, true); } /** @@ -477,7 +480,7 @@ void generic_smp_call_function_single_interrupt(void) * Loop through the call_single_queue and run all the queued callbacks. * Must be called with interrupts disabled. */ -static void __flush_smp_call_function_queue(bool warn_cpu_offline) +static void __flush_smp_call_function_queue(struct pt_regs *regs, bool warn_cpu_offline) { call_single_data_t *csd, *csd_next; struct llist_node *entry, *prev; @@ -543,7 +546,20 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) } csd_lock_record(csd); - csd_do_func(func, info, csd); + /* if (func == rex_terminate) { */ + /* struct termination_data term_data; */ + /* void *data; */ + /* printk("%s sync call to rex_terminate\n", __FILE__); */ + /* term_data.prog = */ + /* info; // we know that bpf termination call */ + /* // will have prog_struct behind the */ + /* // void *info pointer. */ + /* term_data.regs = regs; */ + /* data = &term_data; */ + /* csd_do_func(func, data, csd); */ + /* } else { */ + csd_do_func(func, info, csd); + /* } */ csd_unlock(csd); csd_lock_record(NULL); } else { @@ -574,7 +590,18 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) csd_lock_record(csd); csd_unlock(csd); - csd_do_func(func, info, csd); + /* if (func == rex_terminate) { */ + /* struct termination_data term_data; */ + /* void *data; */ + /* printk("%s !sync call to rex_terminate\n", */ + /* __FILE__); */ + /* term_data.prog = info; */ + /* term_data.regs = regs; */ + /* data = &term_data; */ + /* csd_do_func(func, data, csd); */ + /* } else { */ + csd_do_func(func, info, csd); + /* } */ csd_lock_record(NULL); } else if (type == CSD_TYPE_IRQ_WORK) { irq_work_single(csd); @@ -607,7 +634,7 @@ static void __flush_smp_call_function_queue(bool warn_cpu_offline) * The migration thread has to ensure that an eventually pending wakeup has * been handled before it migrates a task. */ -void flush_smp_call_function_queue(void) +void flush_smp_call_function_queue(struct pt_regs *regs) { unsigned int was_pending; unsigned long flags; @@ -618,7 +645,7 @@ void flush_smp_call_function_queue(void) local_irq_save(flags); /* Get the already pending soft interrupts for RT enabled kernels */ was_pending = local_softirq_pending(); - __flush_smp_call_function_queue(true); + __flush_smp_call_function_queue(regs, true); if (local_softirq_pending()) do_softirq_post_smp_call_flush(was_pending); diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index af7079aa0f36d9..697f9363a1acb3 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -391,6 +391,11 @@ static const struct bpf_func_proto bpf_trace_printk_proto = { .arg2_type = ARG_CONST_SIZE, }; +void rex_trace_printk(void) +{ + trace_bpf_trace_printk(this_cpu_ptr(rex_log_buf)); +} + static void __set_printk_clr_event(struct work_struct *work) { /* diff --git a/meson.build b/meson.build new file mode 100644 index 00000000000000..7b9e8951a54a55 --- /dev/null +++ b/meson.build @@ -0,0 +1,58 @@ +make = find_program('make') +nproc = find_program('nproc') +cpu_count = run_command( + nproc, + capture: true, + check: true +).stdout().strip() + + +kernel_config = custom_target( + 'kernel-config', + output: ['.config'], + command: [ + 'cp', '@SOURCE_ROOT@/scripts/q-script/.config', '@OUTPUT@' + ], + console: true, + build_by_default: false +) + +kbuild_dir = meson.current_build_dir() + +kernel_build = custom_target( + 'kernel-build', + output: ['vmlinux'], + command: [ + 'make', '-C','@SOURCE_ROOT@/linux', '-kj', cpu_count, 'O=' + kbuild_dir + ], + env: ['LLVM=1'], + console: true, + depends: kernel_config, + build_always_stale: true, + build_by_default: false +) + +kernel_libbpf = custom_target( + 'kernel-libbpf', + output: ['libbpf.so', 'bpf_helper_defs.h'], + command: [ + make, '-C', '@SOURCE_ROOT@/linux/tools/lib/bpf/', '-kj', cpu_count, 'O=' + kbuild_dir + ], + env: ['LLVM=1'], + console: true, + depends: kernel_build, + build_by_default: true +) + +subdir('usr') +subdir('tools') + +kernel_dep = declare_dependency( + include_directories: kernel_usr_inc +) + +libbpf_dep = declare_dependency( + link_with: kernel_libbpf[0], + include_directories: kernel_toolslib_inc, + sources: kernel_libbpf[1] +) diff --git a/samples/bpf/.gitignore b/samples/bpf/.gitignore index 0002cd359fb119..ee2c1b25825131 100644 --- a/samples/bpf/.gitignore +++ b/samples/bpf/.gitignore @@ -8,6 +8,7 @@ lwt_len_hist map_perf_test offwaketime per_socket_stats_example +recursive sampleip sock_example sockex1 @@ -37,6 +38,8 @@ tracex4 tracex5 tracex6 tracex7 +tracex8 +tracex9 xdp_adjust_tail xdp_fwd xdp_router_ipv4 @@ -49,3 +52,7 @@ iperf.* /vmlinux.h /bpftool/ /libbpf/ +hello +array_test +hash_test +static_u64_test diff --git a/samples/bpf/Makefile b/samples/bpf/Makefile index 95a4fa1f1e4474..bacc8495834993 100644 --- a/samples/bpf/Makefile +++ b/samples/bpf/Makefile @@ -17,6 +17,7 @@ tprogs-y += tracex3 tprogs-y += tracex4 tprogs-y += tracex5 tprogs-y += tracex6 +tprogs-y += tracex8 tprogs-y += trace_output tprogs-y += lathist tprogs-y += offwaketime @@ -38,6 +39,13 @@ tprogs-y += task_fd_query tprogs-y += ibumad tprogs-y += hbm +tprogs-y += tracex9 + +tprogs-y += recursive +tprogs-y += array_test +tprogs-y += hash_test +tprogs-y += static_u64_test + # Libbpf dependencies LIBBPF_SRC = $(TOOLS_PATH)/lib/bpf LIBBPF_OUTPUT = $(abspath $(BPF_SAMPLES_PATH))/libbpf @@ -58,6 +66,7 @@ tracex3-objs := tracex3_user.o tracex4-objs := tracex4_user.o tracex5-objs := tracex5_user.o $(TRACE_HELPERS) tracex6-objs := tracex6_user.o +tracex8-objs := tracex8_user.o trace_output-objs := trace_output_user.o lathist-objs := lathist_user.o offwaketime-objs := offwaketime_user.o $(TRACE_HELPERS) @@ -81,8 +90,16 @@ hbm-objs := hbm.o $(CGROUP_HELPERS) xdp_router_ipv4-objs := xdp_router_ipv4_user.o $(XDP_SAMPLE) +recursive-objs := recursive_user.o +array_test-objs := array_test_user.o +hash_test-objs := hash_test_user.o +static_u64_test-objs := static_u64_test_user.o +tracex9-objs := tracex9_user.o $(TRACE_HELPERS) + # Tell kbuild to always build the programs always-y := $(tprogs-y) +always-y += hello_kern.o +always-y += tracex9_kern.o always-y += sockex1_kern.o always-y += sockex2_kern.o always-y += sockex3_kern.o @@ -91,6 +108,7 @@ always-y += tracex3.bpf.o always-y += tracex4.bpf.o always-y += tracex5.bpf.o always-y += tracex6.bpf.o +always-y += tracex8_kern.o always-y += trace_output.bpf.o always-y += tcbpf1_kern.o always-y += tc_l2_redirect_kern.o @@ -126,6 +144,11 @@ always-y += hbm_edt_kern.o COMMON_CFLAGS = $(TPROGS_USER_CFLAGS) TPROGS_LDFLAGS = $(TPROGS_USER_LDFLAGS) +always-y += recursive_kern.o +always-y += array_test_kern.o +always-y += hash_test_kern.o +always-y += static_u64_test_kern.o + ifeq ($(ARCH), arm) # Strip all except -D__LINUX_ARM_ARCH__ option needed to handle linux # headers when arm instruction set identification is requested. diff --git a/samples/bpf/array_test_kern.c b/samples/bpf/array_test_kern.c new file mode 100644 index 00000000000000..14e72845d2a44f --- /dev/null +++ b/samples/bpf/array_test_kern.c @@ -0,0 +1,34 @@ + +#include +#include + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(max_entries, 5000); + __type(key, int); + __type(value, int); +} my_map SEC(".maps"); + +SEC("kprobe/kprobe_target_func") +int bpf_prog1(void) +{ + int key = 2500; + __u64 start_time; + __u64 end_time; + __u64 duration; + + int value = bpf_get_prandom_u32(); + bpf_map_update_elem(&my_map, &key, &value, BPF_ANY); + + start_time = bpf_ktime_get_ns(); + bpf_map_lookup_elem(&my_map, &key); + end_time = bpf_ktime_get_ns(); + + duration = end_time - start_time; + + // Print the duration in nanoseconds + bpf_printk("Time elapsed: %llu", duration); + return 0; +} diff --git a/samples/bpf/array_test_user.c b/samples/bpf/array_test_user.c new file mode 100644 index 00000000000000..c3c21c8d6d2dc8 --- /dev/null +++ b/samples/bpf/array_test_user.c @@ -0,0 +1,46 @@ +#define _GNU_SOURCE + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct bpf_link *link = NULL; + struct bpf_program *prog; + struct bpf_object *obj; + char filename[256]; + + snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); + obj = bpf_object__open_file(filename, NULL); + if (libbpf_get_error(obj)) { + fprintf(stderr, "ERROR: opening BPF object file failed\n"); + return 0; + } + + prog = bpf_object__find_program_by_name(obj, "bpf_prog1"); + if (!prog) { + fprintf(stderr, "ERROR: finding a prog in obj file failed\n"); + goto cleanup; + } + + /* load BPF program */ + if (bpf_object__load(obj)) { + fprintf(stderr, "ERROR: loading BPF object file failed\n"); + goto cleanup; + } + + link = bpf_program__attach(prog); + if (libbpf_get_error(link)) { + fprintf(stderr, "ERROR: bpf_program__attach failed\n"); + link = NULL; + goto cleanup; + } + + bpf_link__pin(link, "/sys/fs/bpf/map_test"); + +cleanup: + bpf_link__destroy(link); + bpf_object__close(obj); + return 0; +} diff --git a/samples/bpf/hash_test_kern.c b/samples/bpf/hash_test_kern.c new file mode 100644 index 00000000000000..64acc069b8b9c9 --- /dev/null +++ b/samples/bpf/hash_test_kern.c @@ -0,0 +1,34 @@ + +#include +#include + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 5000); + __type(key, int); + __type(value, int); +} my_map SEC(".maps"); + +SEC("kprobe/kprobe_target_func") +int bpf_prog1(void) +{ + int key = 2500; + __u64 start_time; + __u64 end_time; + __u64 duration; + + int value = bpf_get_prandom_u32(); + bpf_map_update_elem(&my_map, &key, &value, BPF_ANY); + + start_time = bpf_ktime_get_ns(); + bpf_map_lookup_elem(&my_map, &key); + end_time = bpf_ktime_get_ns(); + + duration = end_time - start_time; + + // Print the duration in nanoseconds + bpf_printk("Time elapsed: %llu", duration); + return 0; +} diff --git a/samples/bpf/hash_test_user.c b/samples/bpf/hash_test_user.c new file mode 100644 index 00000000000000..c3c21c8d6d2dc8 --- /dev/null +++ b/samples/bpf/hash_test_user.c @@ -0,0 +1,46 @@ +#define _GNU_SOURCE + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct bpf_link *link = NULL; + struct bpf_program *prog; + struct bpf_object *obj; + char filename[256]; + + snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); + obj = bpf_object__open_file(filename, NULL); + if (libbpf_get_error(obj)) { + fprintf(stderr, "ERROR: opening BPF object file failed\n"); + return 0; + } + + prog = bpf_object__find_program_by_name(obj, "bpf_prog1"); + if (!prog) { + fprintf(stderr, "ERROR: finding a prog in obj file failed\n"); + goto cleanup; + } + + /* load BPF program */ + if (bpf_object__load(obj)) { + fprintf(stderr, "ERROR: loading BPF object file failed\n"); + goto cleanup; + } + + link = bpf_program__attach(prog); + if (libbpf_get_error(link)) { + fprintf(stderr, "ERROR: bpf_program__attach failed\n"); + link = NULL; + goto cleanup; + } + + bpf_link__pin(link, "/sys/fs/bpf/map_test"); + +cleanup: + bpf_link__destroy(link); + bpf_object__close(obj); + return 0; +} diff --git a/samples/bpf/hello_kern.c b/samples/bpf/hello_kern.c new file mode 100644 index 00000000000000..2d7184275638a9 --- /dev/null +++ b/samples/bpf/hello_kern.c @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Refer to samples/bpf/tcp_bpf.readme for the instructions on + * how to run this sample program. + */ +#include + +#include +#include + +int _version SEC("version") = 1; +char _license[] SEC("license") = "GPL"; + +SEC("hello") +int _hello() +{ + bpf_printk("hello bpf\n"); + + return 1; +} diff --git a/samples/bpf/recursive_kern.c b/samples/bpf/recursive_kern.c new file mode 100644 index 00000000000000..14565c258a5f87 --- /dev/null +++ b/samples/bpf/recursive_kern.c @@ -0,0 +1,55 @@ +// Referenced from bpf/samples/hello_kern.c +#include +#include +#include + +#define noinline __attribute__((__noinline__)) + +static __u32 n; + +struct { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __type(key, u32); + __type(value, u32); + __uint(max_entries, 1); +} prog_map SEC(".maps"); + +static noinline int purgatory(struct pt_regs *ctx) +{ + bpf_tail_call(ctx, &prog_map, 0); + + /* bpf_printk("tailcall failed in purgatory\n"); */ + return 0; +} + +SEC("kprobe/") +int calculate_tail_factorial(struct pt_regs *ctx) +{ + /* Base case */ + if(!n) + return 0; + + /* Else, make tail call */ + n -= 1; + bpf_tail_call(ctx, &prog_map, 0); + + /* bpf_printk("tailcall failed in factorial\n"); */ + return 0; +} + +SEC("kprobe/kprobe_target_func") +int bpf_recursive(struct pt_regs *ctx) +{ + int ret = 0; + /* n = ctx->rdi */ + n = (__u32)ctx->di; + + __u64 start = bpf_ktime_get_ns(); + purgatory(ctx); + __u64 stop = bpf_ktime_get_ns(); + + bpf_printk("Time: %llu", stop - start); + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/samples/bpf/recursive_user.c b/samples/bpf/recursive_user.c new file mode 100644 index 00000000000000..2e79208424b0d6 --- /dev/null +++ b/samples/bpf/recursive_user.c @@ -0,0 +1,69 @@ +// Referenced from bpf/samples/hello_user.c +#include +#include +#include +#include +#include "trace_helpers.h" + +int main(int argc, char **argv) +{ + struct bpf_link *link = NULL; + struct bpf_program *prog; + struct bpf_object *obj; + struct bpf_map *prog_map; + char buf[128] = { 0 }; + + snprintf(buf, sizeof(buf), "%s_kern.o", argv[0]); + obj = bpf_object__open(buf); + if (libbpf_get_error(obj)) { + fprintf(stderr, "ERROR: opening BPF object file failed\n"); + return 0; + } + + if (bpf_object__load(obj)) { + fprintf(stderr, "ERROR: loading BPF object file failed\n"); + goto cleanup; + } + + prog_map = bpf_object__find_map_by_name(obj, "prog_map"); + if (libbpf_get_error(prog_map)) { + fprintf(stderr, "ERROR: Could not find map: prog_map\n"); + goto cleanup; + } + + int prog_map_fd = bpf_object__find_map_fd_by_name(obj, "prog_map"); + + prog = bpf_object__find_program_by_name(obj, "calculate_tail_factorial"); + if (!prog) { + fprintf(stderr, "ERROR: finding a prog in obj file failed\n"); + goto cleanup; + } + int prog_fd_idx = 0; + int prog_fd = bpf_program__fd(prog); + if (bpf_map_update_elem(prog_map_fd, &prog_fd_idx, &prog_fd, BPF_ANY) < 0) { + fprintf(stderr, "ERROR: updating prog array failed\n"); + goto cleanup; + } + + prog = bpf_object__find_program_by_name(obj, "bpf_recursive"); + if (!prog) { + fprintf(stderr, "ERROR: finding a prog in obj file failed\n"); + goto cleanup; + } + + link = bpf_program__attach(prog); + if (libbpf_get_error(link)) { + fprintf(stderr, "ERROR: bpf_program__attach failed\n"); + link = NULL; + goto cleanup; + } + + bpf_link__pin(link, "/sys/fs/bpf/recursive_link"); + bpf_object__pin(obj, "/sys/fs/bpf/recursive_obj"); + +cleanup: + bpf_link__destroy(link); + bpf_object__close(obj); + return 0; +} + diff --git a/samples/bpf/static_u64_test_kern.c b/samples/bpf/static_u64_test_kern.c new file mode 100644 index 00000000000000..daa09a025d223d --- /dev/null +++ b/samples/bpf/static_u64_test_kern.c @@ -0,0 +1,42 @@ + +#include +#include + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; + +static __u64 global = 0; + +int bpf_prog2(void) +{ + + // Print the duration in nanoseconds + global += 1; + bpf_printk("Time elapsed: %llu", global); + // bpf_printk("value: %u", lookup); + return 0; +} + +SEC("kprobe/kprobe_target_func") +int bpf_prog1(void) +{ + __u64 start_time; + __u64 end_time; + __u64 duration; + __u64 val; + + // int value = bpf_get_prandom_u32(); + // bpf_map_update_elem(&my_map, &key, &value, BPF_ANY); + // global += 1; + // asm volatile(""::: "memory"); + + start_time = bpf_ktime_get_ns(); + val = global; + end_time = bpf_ktime_get_ns(); + + duration = end_time - start_time; + + // Print the duration in nanoseconds + bpf_printk("Time elapsed: %llu %llu", duration, val); + // bpf_printk("value: %u", lookup); + return 0; +} diff --git a/samples/bpf/static_u64_test_user.c b/samples/bpf/static_u64_test_user.c new file mode 100644 index 00000000000000..c3c21c8d6d2dc8 --- /dev/null +++ b/samples/bpf/static_u64_test_user.c @@ -0,0 +1,46 @@ +#define _GNU_SOURCE + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct bpf_link *link = NULL; + struct bpf_program *prog; + struct bpf_object *obj; + char filename[256]; + + snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); + obj = bpf_object__open_file(filename, NULL); + if (libbpf_get_error(obj)) { + fprintf(stderr, "ERROR: opening BPF object file failed\n"); + return 0; + } + + prog = bpf_object__find_program_by_name(obj, "bpf_prog1"); + if (!prog) { + fprintf(stderr, "ERROR: finding a prog in obj file failed\n"); + goto cleanup; + } + + /* load BPF program */ + if (bpf_object__load(obj)) { + fprintf(stderr, "ERROR: loading BPF object file failed\n"); + goto cleanup; + } + + link = bpf_program__attach(prog); + if (libbpf_get_error(link)) { + fprintf(stderr, "ERROR: bpf_program__attach failed\n"); + link = NULL; + goto cleanup; + } + + bpf_link__pin(link, "/sys/fs/bpf/map_test"); + +cleanup: + bpf_link__destroy(link); + bpf_object__close(obj); + return 0; +} diff --git a/samples/bpf/tracex8_kern.c b/samples/bpf/tracex8_kern.c new file mode 100644 index 00000000000000..2baa50325cf865 --- /dev/null +++ b/samples/bpf/tracex8_kern.c @@ -0,0 +1,9 @@ +#include + +SEC("kprobe/kprobe_target_func") +int bpf_prog1(struct pt_regs *ctx) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; \ No newline at end of file diff --git a/samples/bpf/tracex8_user.c b/samples/bpf/tracex8_user.c new file mode 100644 index 00000000000000..118f80510c9441 --- /dev/null +++ b/samples/bpf/tracex8_user.c @@ -0,0 +1,47 @@ +#define _GNU_SOURCE + +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct bpf_link *link = NULL; + struct bpf_program *prog; + struct bpf_object *obj; + char filename[256]; + int ret = 0; + + snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); + obj = bpf_object__open_file(filename, NULL); + if (libbpf_get_error(obj)) { + fprintf(stderr, "ERROR: opening BPF object file failed\n"); + return 0; + } + + prog = bpf_object__find_program_by_name(obj, "bpf_prog1"); + if (!prog) { + fprintf(stderr, "ERROR: finding a prog in obj file failed\n"); + goto cleanup; + } + + /* load BPF program */ + if (bpf_object__load(obj)) { + fprintf(stderr, "ERROR: loading BPF object file failed\n"); + goto cleanup; + } + + link = bpf_program__attach(prog); + if (libbpf_get_error(link)) { + fprintf(stderr, "ERROR: bpf_program__attach failed\n"); + link = NULL; + goto cleanup; + } + + bpf_link__pin(link, "/sys/fs/bpf/kprobe_link"); + +cleanup: + bpf_link__destroy(link); + bpf_object__close(obj); + return ret ? 0 : 1; +} diff --git a/samples/bpf/tracex9_kern.c b/samples/bpf/tracex9_kern.c new file mode 100644 index 00000000000000..9a4d3928d95944 --- /dev/null +++ b/samples/bpf/tracex9_kern.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include + +struct MapEntry { + u64 data; + struct bpf_spin_lock lock; +}; + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, u32); + __type(value, struct MapEntry); + __uint(max_entries, 256); +} map_array SEC(".maps"); + +SEC("xdp") +int bpf_prog1(struct xdp_md *ctx) +{ + u64 start, end; + u32 key = 0; + struct MapEntry *entry = bpf_map_lookup_elem(&map_array, &key); + + if (entry) { + start = bpf_ktime_get_ns(); + bpf_spin_lock(&entry->lock); + bpf_spin_unlock(&entry->lock); + end = bpf_ktime_get_ns(); + bpf_printk("Spinlock lock and unlock: %llu ns", end - start); + } else { + bpf_printk("Unable to look up map"); + } + return XDP_PASS; +} + +char _license[] SEC("license") = "GPL"; diff --git a/samples/bpf/tracex9_user.c b/samples/bpf/tracex9_user.c new file mode 100644 index 00000000000000..c925bd787650e1 --- /dev/null +++ b/samples/bpf/tracex9_user.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +int main(int argc, char **argv) +{ + struct bpf_link *link = NULL; + struct bpf_program *prog; + struct bpf_object *obj; + + int interface_idx = atoi(argv[1]); + unsigned int xdp_flags = 0; + /* xdp_flags |= XDP_FLAGS_SKB_MODE; */ + xdp_flags |= XDP_FLAGS_DRV_MODE; + + obj = bpf_object__open_file("tracex9_kern.o", NULL); + if (libbpf_get_error(obj)) { + fprintf(stderr, "ERROR: opening BPF object file failed\n"); + return 0; + } + + /* load BPF program */ + if (bpf_object__load(obj)) { + fprintf(stderr, "ERROR: loading BPF object file failed\n"); + goto cleanup; + } + + prog = bpf_object__find_program_by_name(obj, "bpf_prog1"); + if (!prog) { + printf("finding a prog in obj file failed\n"); + goto cleanup; + } + int xdp_main_prog_fd = bpf_program__fd(prog); + + if (bpf_xdp_attach(interface_idx, xdp_main_prog_fd, xdp_flags, NULL) < + 0) { + fprintf(stderr, "ERROR: xdp failed"); + } + +cleanup: + bpf_link__destroy(link); + bpf_object__close(obj); + return 0; +} diff --git a/samples/kprobes/Makefile b/samples/kprobes/Makefile index e774592718d635..f217166848e371 100644 --- a/samples/kprobes/Makefile +++ b/samples/kprobes/Makefile @@ -3,4 +3,5 @@ # then to use one (as root): insmod obj-$(CONFIG_SAMPLE_KPROBES) += kprobe_example.o +obj-$(CONFIG_SAMPLE_KPROBES) += kprobe_target.o obj-$(CONFIG_SAMPLE_KRETPROBES) += kretprobe_example.o diff --git a/samples/kprobes/kprobe_target.c b/samples/kprobes/kprobe_target.c new file mode 100644 index 00000000000000..7732bfe8b0fc14 --- /dev/null +++ b/samples/kprobes/kprobe_target.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#define pr_fmt(fmt) "%s: " fmt, __func__ + +#include +#include + +#define PROC_FILE_NAME "kprobe_target" + +unsigned long noinline kprobe_target_func(unsigned long arg); +unsigned long noinline kprobe_target_func_irq(unsigned long arg); + +enum kprobe_target_cmd { + KPROBE_TARGET_RUN_FUNC = 1313ULL, + KPROBE_TARGET_RUN_FUNC_IRQ = 1314ULL, +}; + + + +/* Defined as global to force standard calling convention */ +unsigned long noinline kprobe_target_func(unsigned long arg) +{ + barrier(); + return arg; +} + +/* Defined as global to force standard calling convention */ +unsigned long noinline kprobe_target_func_irq(unsigned long arg) +{ + barrier(); + task_pid_vnr(current); + return arg; +} + +static long target_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case KPROBE_TARGET_RUN_FUNC: + return kprobe_target_func(arg); + case KPROBE_TARGET_RUN_FUNC_IRQ: + return kprobe_target_func_irq(arg); + default: + return -ENOSYS; + } +} + +struct proc_ops target_ops = { + .proc_flags = PROC_ENTRY_PERMANENT, + .proc_ioctl = target_ioctl, +}; + +static int __init target_init(void) +{ + if (!proc_create(PROC_FILE_NAME, 0600, NULL, &target_ops)) + return -ENOMEM; + + return 0; +} + +static void __exit target_exit(void) +{ + remove_proc_entry(PROC_FILE_NAME, NULL); +} + +module_init(target_init) +module_exit(target_exit) +MODULE_DESCRIPTION("sample kernel module for benchmarking Rex kprobe programs"); +MODULE_LICENSE("GPL"); diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 6daf19809ca4a3..eec83fcc7dfd24 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -698,6 +698,20 @@ static int do_show(int argc, char **argv) return err; } +static int do_terminate(int argc, char**argv) +{ + int prog_id; + int res; + if (argc==0) + return BAD_ARG(); + + prog_id = atoi(argv[0]); + res = bpf_prog_terminate(prog_id); // goes to : tools/lib/bpf/bpf.c + + + return res; +} + static int prog_dump(struct bpf_prog_info *info, enum dump_mode mode, char *filepath, bool opcodes, bool visual, bool linum) @@ -2537,6 +2551,7 @@ static int do_profile(int argc, char **argv) return err; } + #endif /* BPFTOOL_WITHOUT_SKELETONS */ static int do_help(int argc, char **argv) @@ -2547,7 +2562,7 @@ static int do_help(int argc, char **argv) } fprintf(stderr, - "Usage: %1$s %2$s { show | list } [PROG]\n" + "Usage: %1$s %2$s { show | list | terminate } [PROG]\n" " %1$s %2$s dump xlated PROG [{ file FILE | [opcodes] [linum] [visual] }]\n" " %1$s %2$s dump jited PROG [{ file FILE | [opcodes] [linum] }]\n" " %1$s %2$s pin PROG FILE\n" @@ -2609,6 +2624,7 @@ static const struct cmd cmds[] = { { "tracelog", do_tracelog_any }, { "run", do_run }, { "profile", do_profile }, + { "terminate", do_terminate }, { 0 } }; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index 5e38b4887de6ae..ffa425086580d0 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -992,7 +992,12 @@ enum bpf_cmd { BPF_PROG_BIND_MAP, BPF_TOKEN_CREATE, BPF_PROG_STREAM_READ_BY_FD, + BPF_PROG_LOAD_REX_BASE, + BPF_PROG_LOAD_REX, + BPF_SCHED_EXT_ATTACH_REX, + BPF_SCHED_EXT_DETACH_REX, BPF_PROG_ASSOC_STRUCT_OPS, + BPF_PROG_TERMINATE, __MAX_BPF_CMD, }; @@ -1507,6 +1512,11 @@ enum { BPF_STREAM_STDERR = 2, }; +struct rex_sched_ops_sym { + const char __user *name; + __u64 offset; +}; + union bpf_attr { struct { /* anonymous struct used by BPF_MAP_CREATE command */ __u32 map_type; /* one of enum bpf_map_type */ @@ -1595,6 +1605,7 @@ union bpf_attr { __u32 prog_flags; char prog_name[BPF_OBJ_NAME_LEN]; __u32 prog_ifindex; /* ifindex of netdev to prep for */ + __u64 unwinder_insn_off; /* For some prog types expected attach type must be known at * load time to verify attach type specific parts of prog * (context accesses, allowed helpers, etc). @@ -1916,6 +1927,17 @@ union bpf_attr { __u32 prog_fd; } prog_stream_read; + struct { /* BPF_SCHED_EXT_ATTACH_REX */ + __u32 base_prog_fd; + __aligned_u64 sched_ops_syms; /* ptr to rex_sched_ops_sym array */ + __u32 nr_sched_ops_syms; + __u32 timeout_ms; /* ops.timeout_ms, 0 = default */ + __u32 exit_dump_len; /* ops.exit_dump_len, 0 = default */ + __u32 pad; + __aligned_u64 ops_flags; /* SCX_OPS_* flags */ + char name[128]; /* ops.name; empty string = use base->aux->name */ + } sched_ext_attach; + struct { __u32 map_fd; __u32 prog_fd; diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 5846de3642090a..249b1d3aaca905 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -523,6 +523,18 @@ int bpf_map_freeze(int fd) return libbpf_err_errno(ret); } +int bpf_prog_terminate(int prog_id) +{ + union bpf_attr attr; + int res; + memset(&attr, 0, sizeof(attr)); + attr.prog_id = prog_id; + printf("Calling bpf terminate from bpftool : bpf/bpf.c\n"); + res = sys_bpf(BPF_PROG_TERMINATE, &attr, + sizeof(attr)); // goes to : kernel/bpf/syscall.c + return res; +} + static int bpf_map_batch_common(int cmd, int fd, void *in_batch, void *out_batch, void *keys, void *values, __u32 *count, diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 2c8e88ddb67498..c1d771dd88e46d 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -165,6 +165,7 @@ LIBBPF_API int bpf_map_delete_elem(int fd, const void *key); LIBBPF_API int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags); LIBBPF_API int bpf_map_get_next_key(int fd, const void *key, void *next_key); LIBBPF_API int bpf_map_freeze(int fd); +LIBBPF_API int bpf_prog_terminate(int prog_id); /* the new API to terminate a long runnning program */ struct bpf_map_batch_opts { size_t sz; /* size of this struct for forward/backward compatibility */ diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 0be7017800feeb..5dc725f3a5beb1 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -9932,6 +9932,11 @@ static const struct bpf_sec_def section_defs[] = { SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), }; +const struct bpf_sec_defs global_bpf_section_defs LIBBPF_API = { + .arr = section_defs, + .size = ARRAY_SIZE(section_defs), +}; + int libbpf_register_prog_handler(const char *sec, enum bpf_prog_type prog_type, enum bpf_attach_type exp_attach_type, diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index dfc37a6155786f..5125972abc72d1 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -2021,6 +2021,14 @@ LIBBPF_API int libbpf_register_prog_handler(const char *sec, */ LIBBPF_API int libbpf_unregister_prog_handler(int handler_id); +struct bpf_sec_def; +struct bpf_sec_defs { + const struct bpf_sec_def *arr; + size_t size; +}; + +extern const struct bpf_sec_defs global_bpf_section_defs; + #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index d18fbcea7578d5..897f0c4eeaedb4 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -429,6 +429,8 @@ LIBBPF_1.5.0 { bpf_program__attach_sockmap; ring__consume_n; ring_buffer__consume_n; + global_bpf_section_defs; + bpf_prog_terminate; } LIBBPF_1.4.0; LIBBPF_1.6.0 { diff --git a/tools/lib/meson.build b/tools/lib/meson.build new file mode 100644 index 00000000000000..5f167f558f3afd --- /dev/null +++ b/tools/lib/meson.build @@ -0,0 +1 @@ +kernel_toolslib_inc = include_directories('.') diff --git a/tools/meson.build b/tools/meson.build new file mode 100644 index 00000000000000..c2f563b5993f66 --- /dev/null +++ b/tools/meson.build @@ -0,0 +1 @@ +subdir('lib') diff --git a/tools/objtool/noreturns.h b/tools/objtool/noreturns.h index 14f8ab653449c4..10ceec5f6dfbab 100644 --- a/tools/objtool/noreturns.h +++ b/tools/objtool/noreturns.h @@ -11,6 +11,7 @@ NORETURN(__ia32_sys_exit) NORETURN(__ia32_sys_exit_group) NORETURN(__kunit_abort) NORETURN(__module_put_and_kthread_exit) +NORETURN(__rex_landingpad) NORETURN(__stack_chk_fail) NORETURN(__tdx_hypercall_failed) NORETURN(__ubsan_handle_builtin_unreachable) @@ -43,6 +44,8 @@ NORETURN(vpanic) NORETURN(panic_smp_self_stop) NORETURN(rest_init) NORETURN(rewind_stack_and_make_dead) +NORETURN(rex_landingpad) +NORETURN(rex_landingpad_asm) NORETURN(rust_begin_unwind) NORETURN(rust_helper_BUG) NORETURN(sev_es_terminate) diff --git a/usr/include/Makefile b/usr/include/Makefile index 6d86a53c6f0a3d..0c0becdb9a6da5 100644 --- a/usr/include/Makefile +++ b/usr/include/Makefile @@ -168,4 +168,4 @@ $(obj)/%.hdrtest: $(obj)/%.h $(src)/headers_check.pl FORCE # Since GNU Make 4.3, $(patsubst $(obj)/%/,%,$(wildcard $(obj)/*/)) works. # To support older Make versions, use a somewhat tedious way. -clean-files += $(filter-out Makefile headers_check.pl, $(notdir $(wildcard $(obj)/*))) +clean-files += $(filter-out Makefile headers_check.pl meson.build, $(notdir $(wildcard $(obj)/*))) diff --git a/usr/include/meson.build b/usr/include/meson.build new file mode 100644 index 00000000000000..0460739c366a93 --- /dev/null +++ b/usr/include/meson.build @@ -0,0 +1 @@ +kernel_usr_inc = include_directories('.') diff --git a/usr/meson.build b/usr/meson.build new file mode 100644 index 00000000000000..5d13ec5c0e1ec2 --- /dev/null +++ b/usr/meson.build @@ -0,0 +1 @@ +subdir('include')