From 0b5d07e09705d37a3d415d4396eaa1126abf22a6 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:00:27 +0200 Subject: [PATCH 01/16] feat(memtrack): capture allocation stacks in eBPF Copy the caller's user stack in chunks at allocator entry and fold an FNV-1a digest over it in the kernel. The digest rides on the allocation event as stack_hash; the copied bytes, a DWARF-numbered register snapshot and a frame-pointer walk are emitted once per distinct digest on a dedicated ring buffer, so unwinding and symbolication can happen offline. Capture stays off until userspace sets the rodata toggle, so allocator probes are unchanged by default. Refs COD-3222 --- crates/memtrack/src/ebpf/c/allocator.h | 60 ++-- crates/memtrack/src/ebpf/c/event.h | 53 +++- crates/memtrack/src/ebpf/c/main.bpf.c | 1 + .../memtrack/src/ebpf/c/stack_capture.bpf.h | 268 ++++++++++++++++++ .../memtrack/src/ebpf/c/utils/event_helpers.h | 14 +- 5 files changed, 363 insertions(+), 33 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/stack_capture.bpf.h diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 9a4cc238..25a3014c 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -5,24 +5,28 @@ #include "utils/map_helpers.h" #include "utils/process_tracking.h" -#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ - BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ - SEC(UPROBE_SEC) \ - int uprobe_##name(struct pt_regs* ctx) { \ - return store_param(&name##_arg, arg_expr); \ - } \ - SEC(URETPROBE_SEC) \ - int uretprobe_##name(struct pt_regs* ctx) { \ - __u64* arg_ptr = take_param(&name##_arg); \ - if (!arg_ptr) { \ - return 0; \ - } \ - __u64 ret_val = PT_REGS_RC(ctx); \ - if (ret_val == 0) { \ - return 0; \ - } \ - __u64 arg0 = *arg_ptr; \ - submit_block; \ +#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ + BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ + SEC(UPROBE_SEC) \ + int uprobe_##name(struct pt_regs* ctx) { \ + capture_stack(ctx); \ + return store_param(&name##_arg, arg_expr); \ + } \ + SEC(URETPROBE_SEC) \ + int uretprobe_##name(struct pt_regs* ctx) { \ + __u64* arg_ptr = take_param(&name##_arg); \ + if (!arg_ptr) { \ + return 0; \ + } \ + /* The slot is per-thread but shared by allocators, so every return must \ + * clear it before failure can strand a nested call's identity. */ \ + __u64 stack_hash = take_stack_hash(); \ + __u64 ret_val = PT_REGS_RC(ctx); \ + if (ret_val == 0) { \ + return 0; \ + } \ + __u64 arg0 = *arg_ptr; \ + submit_block; \ } #define UPROBE_RET(name, arg_expr, submit_block) \ @@ -50,6 +54,8 @@ return 0; \ } \ \ + capture_stack(ctx); \ + \ struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr}; \ \ bpf_map_update_elem(&name##_args, &tid, &args, BPF_ANY); \ @@ -63,6 +69,7 @@ if (!args) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ \ struct name##_args_t a = *args; \ bpf_map_delete_elem(&name##_args, &tid); \ @@ -77,20 +84,22 @@ submit_block; \ } -UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), + { return submit_alloc_event(arg0, ret_val, stack_hash); }) UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); }) UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx), - { return submit_calloc_event(arg0, ret_val); }) + { return submit_calloc_event(arg0, ret_val, stack_hash); }) UPROBE_ARGS_RET(realloc, PT_REGS_PARM2(ctx), PT_REGS_PARM1(ctx), - { return submit_realloc_event(arg1, ret_val, arg0); }) + { return submit_realloc_event(arg1, ret_val, arg0, stack_hash); }) UPROBE_ARG_RET(aligned_alloc, PT_REGS_PARM2(ctx), - { return submit_aligned_alloc_event(arg0, ret_val); }) + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), { return submit_aligned_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) /* * posix_memalign(void** memptr, size_t alignment, size_t size) @@ -115,6 +124,8 @@ int uprobe_posix_memalign(struct pt_regs* ctx) { return 0; } + capture_stack(ctx); + struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)}; bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY); return 0; @@ -127,6 +138,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { if (!args) { return 0; } + __u64 stack_hash = take_stack_hash(); struct posix_memalign_args_t a = *args; bpf_map_delete_elem(&posix_memalign_args, &tid); @@ -140,7 +152,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { return 0; } - return submit_aligned_alloc_event(a.size, addr); + return submit_aligned_alloc_event(a.size, addr, stack_hash); } struct mmap_args { diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index bf0677c9..8401d518 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -15,6 +15,47 @@ #define EVENT_TYPE_RSS 12 #define EVENT_TYPE_RMAP 13 +/* Largest user-stack copy one definition can carry. The scratch buffer holding + * header plus bytes is a per-CPU map value, capped at PCPU_MIN_UNIT_SIZE + * (32 KiB) by the kernel allocator. */ +#define MEMTRACK_MAX_STACK_COPY (32 * 1024 - 512) + +/* Registers, indexed by the capturing architecture's DWARF register number + * (x86_64: 0=rax .. 7=rsp, 8..15=r8-r15, 16=rip; aarch64: 0..30=x0-x30, + * 31=sp, 32=pc). Slots the architecture does not define stay zero. An offline + * DWARF unwinder needs the callee-saved ones to evaluate CFA rules, not just + * ip/sp/bp. */ +#define MEMTRACK_STACK_REGS 33 + +/* Counter slots in the stack_counters array map. */ +#define MEMTRACK_STACK_COUNTER_COPY_FAILED 0 +#define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1 +/* bpf_get_stackid() has several negative outcomes (no user callchain, + * hash-bucket collision, or no free bucket), so this counts only missing ids. */ +#define MEMTRACK_STACK_COUNTER_STACKID_FAILED 2 +#define MEMTRACK_STACK_COUNTER_TRUNCATED 3 +#define MEMTRACK_STACK_COUNTER_RING_FULL 4 +#define MEMTRACK_STACK_COUNTER_PREEMPTED 5 +#define MEMTRACK_STACK_COUNTER_COUNT 6 + +struct stack_regs { + uint64_t reg[MEMTRACK_STACK_REGS]; +}; + +/* Head of a stack-definition record; `copy_len` raw stack bytes read upwards + * from `sp` follow it. */ +struct stack_def_header { + uint64_t hash; + int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ + uint64_t sp; /* user stack pointer the copy starts at */ + uint32_t pid; + uint32_t tid; + uint32_t copy_len; + uint8_t truncated; /* the copy hit the size cap or a short read */ + uint8_t _pad[3]; + struct stack_regs regs; +}; + /* Common header shared by all event types */ struct event_header { uint8_t event_type; /* See EVENT_TYPE_* constants above */ @@ -29,8 +70,9 @@ struct event { union { /* Allocation events (malloc, calloc, aligned_alloc) */ struct { - uint64_t addr; /* address returned */ - uint64_t size; /* size requested */ + uint64_t addr; /* address returned */ + uint64_t size; /* size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } alloc; /* Deallocation event (free) */ @@ -40,9 +82,10 @@ struct event { /* Reallocation event - includes both old and new addresses */ struct { - uint64_t old_addr; /* previous address (can be NULL) */ - uint64_t new_addr; /* new address returned */ - uint64_t size; /* new size requested */ + uint64_t old_addr; /* previous address (can be NULL) */ + uint64_t new_addr; /* new address returned */ + uint64_t size; /* new size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } realloc; /* Memory mapping events (mmap, munmap, brk) */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 5a8d6ff0..b405f572 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -11,6 +11,7 @@ #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" +#include "stack_capture.bpf.h" #include "utils/event_helpers.h" #include "utils/folio.h" #include "utils/map_helpers.h" diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h new file mode 100644 index 00000000..d4168b39 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -0,0 +1,268 @@ +#ifndef __STACK_CAPTURE_BPF_H__ +#define __STACK_CAPTURE_BPF_H__ + +#include "event.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* == Allocation stack capture == + * + * At allocator entry the caller's raw user stack is copied and hashed; the hash + * travels on the allocation event and identifies the call path. The first time + * a hash is seen, the copied bytes plus a full user register snapshot are + * emitted as a stack-definition record so the stack can be DWARF-unwound + * offline. An in-kernel frame-pointer walk rides along in the definition as the + * fallback for binaries whose .eh_frame is missing or whose DWARF unwind + * truncates. + * + * A 64-bit FNV-1a digest means distinct byte sequences alias with birthday + * probability roughly n^2/2^65: negligible at the thousands of identities a + * run produces, but not zero. It also splits one call path into several + * identities whenever the locals and arguments living in the copied region + * differ. That is a deliberate trade: aliasing corrupts attribution, splitting + * only costs definition records. + */ + +const volatile __u8 capture_stacks_enabled = 0; +const volatile __u32 stack_copy_size = 8192; + +#define STACK_TRACE_MAX_DEPTH 127 +/* Chunk granularity of the stack copy: the recovered length is exact to within + * one chunk, and a full-budget copy costs MEMTRACK_MAX_STACK_COPY/chunk helper + * calls. MEMTRACK_MAX_STACK_COPY is a whole multiple of it. */ +#define STACK_COPY_CHUNK 512 +#define FNV64_OFFSET 0xcbf29ce484222325ULL +#define FNV64_PRIME 0x00000100000001b3ULL + +/* Frame-pointer walk results, indexed by the id bpf_get_stackid() returns. */ +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(max_entries, 16384); + __type(key, __u32); + __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); +} stack_traces SEC(".maps"); + +/* Definitions are bulky but rare (one per distinct hash), so they get their own + * ring rather than inflating the fixed-size `struct event` path. */ +BPF_RINGBUF(stack_defs, 64 * 1024 * 1024); +BPF_HASH_MAP(seen_stack_hashes, __u64, __u8, 65536); +BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); +BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); + +/* The definition record is built in place here and handed to the ring buffer as + * one contiguous variable-length blob. `words` aliases `bytes` so the hash loop + * reads whole registers without a per-byte shift chain. */ +struct stack_def_scratch { + struct stack_def_header header; + union { + __u8 bytes[MEMTRACK_MAX_STACK_COPY]; + __u64 words[MEMTRACK_MAX_STACK_COPY / 8]; + }; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct stack_def_scratch); +} stack_scratch SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u32); +} stack_busy SEC(".maps"); + +static __always_inline void bump_stack_counter(__u32 index) { + __u64* slot = bpf_map_lookup_elem(&stack_counters, &index); + if (slot) { + __sync_fetch_and_add(slot, 1); + } +} + +#if defined(__TARGET_ARCH_x86) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + out->reg[0] = ctx->ax; + out->reg[1] = ctx->dx; + out->reg[2] = ctx->cx; + out->reg[3] = ctx->bx; + out->reg[4] = ctx->si; + out->reg[5] = ctx->di; + out->reg[6] = ctx->bp; + out->reg[7] = ctx->sp; + out->reg[8] = ctx->r8; + out->reg[9] = ctx->r9; + out->reg[10] = ctx->r10; + out->reg[11] = ctx->r11; + out->reg[12] = ctx->r12; + out->reg[13] = ctx->r13; + out->reg[14] = ctx->r14; + out->reg[15] = ctx->r15; + out->reg[16] = ctx->ip; +} +#elif defined(__TARGET_ARCH_arm64) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + struct user_pt_regs* uregs = (struct user_pt_regs*)ctx; +#pragma unroll + for (int i = 0; i < 31; i++) { + out->reg[i] = uregs->regs[i]; + } + out->reg[31] = uregs->sp; + out->reg[32] = uregs->pc; +} +#else +#error "stack capture needs a DWARF register mapping for this architecture" +#endif + +/* Copy and hash the caller's stack, stashing the identity for the matching + * uretprobe. Emit a definition on the first occurrence. */ +static __always_inline void capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { + __u32 zero = 0; + struct stack_def_scratch* scratch = bpf_map_lookup_elem(&stack_scratch, &zero); + if (!scratch) { + return; + } + + __u64 sp = PT_REGS_SP(ctx); + __u32 want = stack_copy_size; + if (want > MEMTRACK_MAX_STACK_COPY) { + want = MEMTRACK_MAX_STACK_COPY; + } + want &= ~(__u32)(STACK_COPY_CHUNK - 1); + if (want < STACK_COPY_CHUNK) { + want = STACK_COPY_CHUNK; + } + + /* bpf_probe_read_user() is all-or-nothing and the readable region ends at + * the top of the stack mapping, which is not knowable up front, so the copy + * advances in chunks and stops at the first unreadable one. A failed chunk + * can leave up to STACK_COPY_CHUNK - 1 readable bytes at the top of the + * mapping uncopied, so a complete copy is exact only to chunk granularity. + * + * Each chunk is hashed as it lands, with a constant iteration count the + * compiler fully unrolls. A single loop over the whole copy instead costs + * the verifier a state fork per word and blows the one-million instruction + * budget well before the maximum copy size. */ + __u64 hash = FNV64_OFFSET; + __u32 got = 0; +#pragma clang loop unroll(disable) + for (__u32 off = 0; off + STACK_COPY_CHUNK <= MEMTRACK_MAX_STACK_COPY; + off += STACK_COPY_CHUNK) { + if (off >= want) { + break; + } + if (bpf_probe_read_user(&scratch->bytes[off], STACK_COPY_CHUNK, (void*)(sp + off)) != 0) { + break; + } + + __u32 base = off >> 3; +#pragma unroll + for (__u32 word = 0; word < STACK_COPY_CHUNK / 8; word++) { + hash = (hash ^ scratch->words[base + word]) * FNV64_PRIME; + } + got = off + STACK_COPY_CHUNK; + } + + if (got == 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_COPY_FAILED); + return; + } + + __u8 truncated = got >= want; + if (truncated) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_TRUNCATED); + } + + /* Fold in the length so a truncated prefix of a deep stack cannot collide + * with a full copy of a shallower one, and keep 0 reserved as the "no + * stack" marker on allocation events. */ + hash = (hash ^ got) * FNV64_PRIME; + if (hash == 0) { + hash = FNV64_OFFSET; + } + + __u64 tid = ids.tid; + bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); + + __u8 marker = 1; + long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); + if (gate_result == -17) { + return; + } + if (gate_result != 0) { + /* A full gate cannot retain this identity, so emit it on every + * occurrence rather than make the allocation hash unresolvable. */ + bump_stack_counter(MEMTRACK_STACK_COUNTER_HASH_MAP_FULL); + } + + __s64 stackid = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK); + if (stackid < 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_STACKID_FAILED); + } + + scratch->header.hash = hash; + scratch->header.stackid = stackid; + scratch->header.sp = sp; + scratch->header.pid = ids.tgid; + scratch->header.tid = ids.tid; + scratch->header.copy_len = got; + scratch->header.truncated = truncated; + scratch->header._pad[0] = 0; + scratch->header._pad[1] = 0; + scratch->header._pad[2] = 0; + fill_stack_regs(&scratch->header.regs, ctx); + + if (bpf_ringbuf_output(&stack_defs, scratch, sizeof(struct stack_def_header) + got, 0) != 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_RING_FULL); + bpf_map_delete_elem(&seen_stack_hashes, &hash); + } +} + +static __always_inline void capture_stack(struct pt_regs* ctx) { + if (!capture_stacks_enabled || !is_enabled()) { + return; + } + + struct task_ids ids = current_task_ids(); + if (!is_tracked(ids.tgid)) { + return; + } + + __u32 zero = 0; + __u32* busy = bpf_map_lookup_elem(&stack_busy, &zero); + if (!busy) { + return; + } + if (__sync_val_compare_and_swap(busy, 0, 1) != 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_PREEMPTED); + return; + } + + capture_stack_inner(ctx, ids); + + /* A plain store, not an atomic release: the only contender is a task that + * preempted this one on this same CPU, and the context switch between them + * already orders the write. The BPF backend cannot select a release store. */ + *busy = 0; +} + +/* The identity stashed by the matching entry probe, or 0 when capture is off or + * the entry probe bailed out. */ +static __always_inline __u64 take_stack_hash(void) { + if (!capture_stacks_enabled) { + return 0; + } + + __u64 tid = current_tid(); + __u64* hash = bpf_map_lookup_elem(&pending_stack_hash, &tid); + if (!hash) { + return 0; + } + + __u64 value = *hash; + bpf_map_delete_elem(&pending_stack_hash, &tid); + return value; +} + +#endif /* __STACK_CAPTURE_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index ca5969a9..2cbd83d7 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -2,6 +2,7 @@ #define __EVENT_HELPERS_H__ #include "../event.h" +#include "../stack_capture.bpf.h" #include "map_helpers.h" #include "process_tracking.h" @@ -88,24 +89,27 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(owner.tgid, evt_type, fill_data); \ } -static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_calloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } @@ -113,11 +117,13 @@ static __always_inline int submit_free_event(__u64 addr) { SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); } -static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { +static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size, + __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; + e->data.realloc.stack_hash = stack_hash; }); } From 133cb69d54929a67afff5dcc2ae5417dfbf0d0d8 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:00:53 +0200 Subject: [PATCH 02/16] feat(memtrack): add userspace stack-capture module Add the userspace half of allocation stack capture: env-driven configuration, stack-definition ring parsing, loss counters, per-pid module mapping tracking, a folding recorder that deduplicates definitions and counts occurrences, and the report it produces. Nothing constructs these yet; the tracker wiring follows. Refs COD-3222 --- crates/memtrack/src/ebpf/events.rs | 29 ++++ crates/memtrack/src/ebpf/mod.rs | 4 + crates/memtrack/src/ebpf/stacks/config.rs | 89 +++++++++++ crates/memtrack/src/ebpf/stacks/counters.rs | 62 ++++++++ crates/memtrack/src/ebpf/stacks/events.rs | 138 ++++++++++++++++ crates/memtrack/src/ebpf/stacks/mod.rs | 6 + crates/memtrack/src/ebpf/stacks/modules.rs | 165 ++++++++++++++++++++ crates/memtrack/src/ebpf/stacks/recorder.rs | 157 +++++++++++++++++++ crates/memtrack/src/ebpf/stacks/report.rs | 82 ++++++++++ 9 files changed, 732 insertions(+) create mode 100644 crates/memtrack/src/ebpf/stacks/config.rs create mode 100644 crates/memtrack/src/ebpf/stacks/counters.rs create mode 100644 crates/memtrack/src/ebpf/stacks/events.rs create mode 100644 crates/memtrack/src/ebpf/stacks/mod.rs create mode 100644 crates/memtrack/src/ebpf/stacks/modules.rs create mode 100644 crates/memtrack/src/ebpf/stacks/recorder.rs create mode 100644 crates/memtrack/src/ebpf/stacks/report.rs diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 4ed422a5..a3c76deb 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -110,6 +110,35 @@ pub fn parse_event(data: &[u8]) -> Option { kind, }) } +/// The stack identity an allocation event carries, or `None` for event types +/// that have no allocation-site stack. +pub fn parse_alloc_stack_record(data: &[u8]) -> Option { + if data.len() < std::mem::size_of::() { + return None; + } + + // SAFETY: The data must be a valid `bindings::event`. + let event = unsafe { &*(data.as_ptr() as *const bindings::event) }; + + // SAFETY: The fields must be properly initialized in eBPF. + Some(unsafe { + match event.header.event_type as u32 { + EVENT_TYPE_MALLOC => crate::ebpf::stacks::events::StackRecord::Alloc { + hash: event.data.alloc.stack_hash, + }, + EVENT_TYPE_CALLOC => crate::ebpf::stacks::events::StackRecord::Alloc { + hash: event.data.alloc.stack_hash, + }, + EVENT_TYPE_ALIGNED_ALLOC => crate::ebpf::stacks::events::StackRecord::Alloc { + hash: event.data.alloc.stack_hash, + }, + EVENT_TYPE_REALLOC => crate::ebpf::stacks::events::StackRecord::Alloc { + hash: event.data.realloc.stack_hash, + }, + _ => return None, + } + }) +} /// A request from the exec-mapping watcher to attach allocator probes. #[derive(Debug, Clone, Copy)] diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index ad556519..aa3ceb74 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -4,9 +4,13 @@ mod memtrack; pub(crate) mod poller; mod proc_fs; mod spawn; +mod stacks; mod tracker; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; +pub use stacks::config::StackCaptureConfig; +pub use stacks::counters::StackCaptureStats; +pub use stacks::report::{StackDefinitionReport, StackReport}; pub use tracker::Tracker; diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs new file mode 100644 index 00000000..d5a757af --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/config.rs @@ -0,0 +1,89 @@ +use crate::prelude::*; + +pub struct StackCaptureConfig { + pub copy_size: u32, + /// Frame-pointer walk slots. Exhausting them costs the fallback chain for + /// stacks beyond the limit, never an allocation event. + pub stack_trace_capacity: u32, + pub dump_path: Option, +} + +impl StackCaptureConfig { + pub const DEFAULT_COPY_SIZE: u32 = 8192; + pub const DEFAULT_STACK_TRACE_CAPACITY: u32 = 16384; + + /// Returns `None` unless stack capture was explicitly enabled. + pub fn from_env() -> Option { + if std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS") + .ok() + .as_deref() + != Some("1") + { + return None; + } + + let copy_size = match std::env::var("CODSPEED_MEMTRACK_STACK_COPY_SIZE") { + Ok(value) => match value.parse::() { + Ok(size) => size, + Err(error) => { + warn!( + "Invalid CODSPEED_MEMTRACK_STACK_COPY_SIZE {value:?}: {error}; using default" + ); + Self::DEFAULT_COPY_SIZE + } + }, + Err(std::env::VarError::NotPresent) => Self::DEFAULT_COPY_SIZE, + Err(error) => { + warn!("Invalid CODSPEED_MEMTRACK_STACK_COPY_SIZE: {error}; using default"); + Self::DEFAULT_COPY_SIZE + } + }; + let dump_path = + std::env::var_os("CODSPEED_MEMTRACK_STACK_DUMP").map(std::path::PathBuf::from); + + Some(Self { + copy_size: clamp_copy_size(copy_size), + stack_trace_capacity: Self::DEFAULT_STACK_TRACE_CAPACITY, + dump_path, + }) + } + + pub fn with_copy_size(copy_size: u32) -> Self { + Self { + copy_size: clamp_copy_size(copy_size), + stack_trace_capacity: Self::DEFAULT_STACK_TRACE_CAPACITY, + dump_path: None, + } + } +} + +/// The kernel copies whole chunks, so a budget that is not a multiple of one +/// would hash bytes it never emits. +fn clamp_copy_size(copy_size: u32) -> u32 { + const CHUNK: u32 = 512; + let rounded = copy_size / CHUNK * CHUNK; + rounded.clamp( + CHUNK, + crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY, + ) +} + +#[cfg(test)] +mod tests { + use super::clamp_copy_size; + + #[test] + fn rounds_down_to_a_whole_chunk() { + assert_eq!(clamp_copy_size(8_700), 8_192); + } + + #[test] + fn clamps_to_low_bound() { + assert_eq!(clamp_copy_size(63), 512); + } + + #[test] + fn clamps_to_high_bound() { + assert_eq!(clamp_copy_size(u32::MAX), 32_256); + } +} diff --git a/crates/memtrack/src/ebpf/stacks/counters.rs b/crates/memtrack/src/ebpf/stacks/counters.rs new file mode 100644 index 00000000..a4b5ce70 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/counters.rs @@ -0,0 +1,62 @@ +use crate::prelude::*; + +#[derive(Debug, Clone, Copy, Default, serde::Serialize)] +pub struct StackCaptureStats { + pub copy_failed: u64, + pub hash_map_full: u64, + pub stackid_failed: u64, + pub truncated: u64, + pub ring_full: u64, + /// Captures skipped because another BPF program used the per-CPU scratch. + pub preempted: u64, +} + +impl StackCaptureStats { + pub fn read(map: &impl libbpf_rs::MapCore) -> Result { + Ok(Self { + copy_failed: slot( + map, + crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_COPY_FAILED, + "copy_failed", + )?, + hash_map_full: slot( + map, + crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_HASH_MAP_FULL, + "hash_map_full", + )?, + stackid_failed: slot( + map, + crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_STACKID_FAILED, + "stackid_failed", + )?, + truncated: slot( + map, + crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_TRUNCATED, + "truncated", + )?, + ring_full: slot( + map, + crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_RING_FULL, + "ring_full", + )?, + preempted: slot( + map, + crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_PREEMPTED, + "preempted", + )?, + }) + } +} + +fn slot(map: &impl libbpf_rs::MapCore, index: u32, name: &str) -> Result { + let key = index.to_ne_bytes(); + let value = map + .lookup(&key, libbpf_rs::MapFlags::ANY) + .with_context(|| format!("failed to read {name} counter"))? + .ok_or_else(|| anyhow!("{name} counter slot {index} missing"))?; + let bytes: [u8; 8] = value + .as_slice() + .try_into() + .map_err(|_| anyhow!("{name} counter value has unexpected size"))?; + Ok(u64::from_ne_bytes(bytes)) +} diff --git a/crates/memtrack/src/ebpf/stacks/events.rs b/crates/memtrack/src/ebpf/stacks/events.rs new file mode 100644 index 00000000..ac5f4d4a --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/events.rs @@ -0,0 +1,138 @@ +use crate::prelude::*; + +#[derive(Debug, Clone)] +pub struct StackDefinition { + pub hash: u64, + pub stackid: i64, + pub sp: u64, + pub pid: u32, + pub tid: u32, + pub truncated: bool, + pub regs: [u64; 33], + /// The payload is validated and measured but not retained. + pub copy_len: u32, +} + +#[derive(Debug)] +pub enum StackRecord { + Definition(Box), + Alloc { hash: u64 }, +} + +pub fn parse_stack_definition(data: &[u8]) -> Option { + let header_len = std::mem::size_of::(); + if data.len() < header_len { + warn!( + "malformed stack definition record: got {} bytes, need at least {}", + data.len(), + header_len + ); + return None; + } + + // SAFETY: The length was checked, and the layout is the bindgen-generated C ABI struct. + let header: crate::ebpf::events::bindings::stack_def_header = + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; + let copy_len = header.copy_len as usize; + let Some(record_len) = header_len.checked_add(copy_len) else { + warn!( + "malformed stack definition record: got {} bytes, header {} + copy_len {} overflows", + data.len(), + header_len, + copy_len + ); + return None; + }; + if data.len() < record_len { + warn!( + "malformed stack definition record: got {} bytes, need {}", + data.len(), + record_len + ); + return None; + } + + Some(StackRecord::Definition(Box::new(StackDefinition { + hash: header.hash, + stackid: header.stackid, + sp: header.sp, + pid: header.pid, + tid: header.tid, + truncated: header.truncated != 0, + regs: header.regs.reg, + copy_len: header.copy_len, + }))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ebpf::events::bindings; + + fn encode(header: bindings::stack_def_header, payload: &[u8]) -> Vec { + // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const bindings::stack_def_header).cast::(), + std::mem::size_of::(), + ) + }; + let mut data = header_bytes.to_vec(); + data.extend_from_slice(payload); + data + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let regs = std::array::from_fn(|index| 0x1000 + index as u64); + let header = bindings::stack_def_header { + hash: 0x0123_4567_89ab_cdef, + stackid: -17, + sp: 0x7fff_1234_5000, + pid: 41, + tid: 42, + copy_len: 5, + truncated: 1, + _pad: [0; 3], + regs: bindings::stack_regs { reg: regs }, + }; + let payload = [1, 2, 3, 4, 5]; + + let record = parse_stack_definition(&encode(header, &payload)); + let Some(StackRecord::Definition(definition)) = record else { + panic!("expected stack definition"); + }; + + assert_eq!(definition.hash, header.hash); + assert_eq!(definition.stackid, header.stackid); + assert_eq!(definition.sp, header.sp); + assert_eq!(definition.pid, header.pid); + assert_eq!(definition.tid, header.tid); + assert!(definition.truncated); + assert_eq!(definition.regs, regs); + assert_eq!(definition.copy_len, payload.len() as u32); + } + + #[test] + fn truncated_buffer_returns_none() { + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack_definition(&data).is_none()); + } + + #[test] + fn missing_payload_returns_none() { + let header = bindings::stack_def_header { + hash: 1, + stackid: 2, + sp: 3, + pid: 4, + tid: 5, + copy_len: 4, + truncated: 0, + _pad: [0; 3], + regs: bindings::stack_regs { reg: [0; 33] }, + }; + let data = encode(header, &[1, 2, 3]); + assert!(parse_stack_definition(&data).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/stacks/mod.rs b/crates/memtrack/src/ebpf/stacks/mod.rs new file mode 100644 index 00000000..f41dc8d9 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/mod.rs @@ -0,0 +1,6 @@ +pub mod config; +pub mod counters; +pub mod events; +pub mod modules; +pub mod recorder; +pub mod report; diff --git a/crates/memtrack/src/ebpf/stacks/modules.rs b/crates/memtrack/src/ebpf/stacks/modules.rs new file mode 100644 index 00000000..3559509a --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/modules.rs @@ -0,0 +1,165 @@ +use std::collections::{BTreeMap, HashMap}; +use std::time::{Duration, Instant}; + +const RESCAN_INTERVAL: Duration = Duration::from_millis(250); + +#[derive(Debug, Clone, serde::Serialize)] +pub struct ModuleMapping { + pub start: u64, + pub end: u64, + pub offset: u64, + pub path: String, +} + +pub struct ModuleTracker { + mappings: HashMap>, + last_read: HashMap, +} + +impl ModuleTracker { + pub fn new() -> Self { + Self { + mappings: HashMap::new(), + last_read: HashMap::new(), + } + } + + /// Re-read the pid's mappings at most once per [`RESCAN_INTERVAL`]. Callers + /// on the allocation path use this: a process can `dlopen` at any time, but + /// re-reading `/proc` per captured stack would cost more than it recovers. + pub fn observe(&mut self, pid: u32) { + if self + .last_read + .get(&pid) + .is_some_and(|last| last.elapsed() < RESCAN_INTERVAL) + { + return; + } + self.refresh(pid); + } + + /// Re-read the pid's mappings now. Callers holding the process stopped use + /// this: it is the only moment the mappings are guaranteed readable, and a + /// short-lived process is usually gone by the time its stacks surface. + pub fn refresh(&mut self, pid: u32) { + self.last_read.insert(pid, Instant::now()); + + let Ok(contents) = std::fs::read_to_string(format!("/proc/{pid}/maps")) else { + return; + }; + let by_start = self.mappings.entry(pid).or_default(); + for mapping in parse_maps(&contents) { + by_start.insert(mapping.start, mapping); + } + } + + pub fn snapshot(&self) -> BTreeMap> { + self.mappings + .iter() + .map(|(pid, mappings)| (*pid, mappings.values().cloned().collect())) + .collect() + } +} + +fn parse_maps(contents: &str) -> Vec { + contents.lines().filter_map(parse_map_line).collect() +} + +fn parse_map_line(line: &str) -> Option { + let mut fields = line.split_whitespace(); + let range = fields.next()?; + let permissions = fields.next()?; + let offset = fields.next()?; + let device = fields.next()?; + let inode = fields.next()?; + + if !permissions.contains('x') { + return None; + } + let (start, end) = range.split_once('-')?; + let start = u64::from_str_radix(start, 16).ok()?; + let end = u64::from_str_radix(end, 16).ok()?; + if start >= end { + return None; + } + let offset = u64::from_str_radix(offset, 16).ok()?; + parse_device(device)?; + inode.parse::().ok()?; + + let mut remaining = line; + for _ in 0..5 { + let trimmed = remaining.trim_start(); + let end = trimmed.find(|character: char| character.is_whitespace())?; + remaining = &trimmed[end..]; + } + let path = remaining.trim(); + if !path.starts_with('/') { + return None; + } + + Some(ModuleMapping { + start, + end, + offset, + path: path.to_owned(), + }) +} + +fn parse_device(device: &str) -> Option<(u64, u64)> { + let (major, minor) = device.split_once(':')?; + Some(( + u64::from_str_radix(major, 16).ok()?, + u64::from_str_radix(minor, 16).ok()?, + )) +} + +#[cfg(test)] +mod tests { + use super::parse_maps; + + #[test] + fn keeps_executable_file_mapping() { + let mappings = + parse_maps("55a1b2c00000-55a1b2c21000 r-xp 00001000 08:02 1234 /usr/bin/foo\n"); + assert_eq!(mappings.len(), 1); + assert_eq!(mappings[0].start, 0x55a1b2c00000); + assert_eq!(mappings[0].end, 0x55a1b2c21000); + assert_eq!(mappings[0].offset, 0x1000); + assert_eq!(mappings[0].path, "/usr/bin/foo"); + } + + #[test] + fn drops_non_executable_mapping() { + assert!( + parse_maps("55a1b2c00000-55a1b2c21000 r--p 00000000 08:02 1234 /usr/bin/foo\n") + .is_empty() + ); + } + + #[test] + fn drops_pseudo_and_anonymous_mappings() { + let contents = concat!( + "7f0000000000-7f0000001000 r-xp 00000000 00:00 0 [vdso]\n", + "00600000-00601000 r-xp 00000000 00:00 0 [heap]\n", + "7f0000010000-7f0000020000 r-xp 00000000 00:00 0\n", + ); + assert!(parse_maps(contents).is_empty()); + } + + #[test] + fn preserves_spaces_in_path() { + let mappings = + parse_maps("55a1b2c00000-55a1b2c21000 r-xp 00000000 08:02 1234 /opt/my program/bin\n"); + assert_eq!(mappings.len(), 1); + assert_eq!(mappings[0].path, "/opt/my program/bin"); + } + + #[test] + fn skips_malformed_lines() { + let contents = concat!( + "not a maps line\n", + "55a1b2c00000-55a1b2c21000 r-xp not-hex 08:02 1234 /usr/bin/foo\n", + ); + assert!(parse_maps(contents).is_empty()); + } +} diff --git a/crates/memtrack/src/ebpf/stacks/recorder.rs b/crates/memtrack/src/ebpf/stacks/recorder.rs new file mode 100644 index 00000000..76b27c36 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/recorder.rs @@ -0,0 +1,157 @@ +use crate::prelude::*; + +use parking_lot::Mutex; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::thread::{self, JoinHandle}; + +use super::config::StackCaptureConfig; +use super::counters::StackCaptureStats; +use super::events::{StackDefinition, StackRecord}; +use super::modules::ModuleTracker; +use super::report::{StackDefinitionReport, StackReport}; + +pub struct StackRecorder { + config: StackCaptureConfig, + modules: Arc>, + thread: JoinHandle, +} + +struct FoldingState { + definitions: HashMap, + occurrences: HashMap, + alloc_events_with_stack: u64, + alloc_events_without_stack: u64, + modules: Arc>, +} + +impl FoldingState { + fn new(modules: Arc>) -> Self { + Self { + definitions: HashMap::new(), + occurrences: HashMap::new(), + alloc_events_with_stack: 0, + alloc_events_without_stack: 0, + modules, + } + } + + fn record(&mut self, record: StackRecord) { + match record { + StackRecord::Definition(definition) => { + let definition = *definition; + self.modules.lock().observe(definition.pid); + self.definitions + .entry(definition.hash) + .or_insert(definition); + } + StackRecord::Alloc { hash: 0 } => { + self.alloc_events_without_stack += 1; + } + StackRecord::Alloc { hash } => { + self.alloc_events_with_stack += 1; + *self.occurrences.entry(hash).or_default() += 1; + } + } + } +} + +impl StackRecorder { + /// Spawns the folding thread. The returned sender is cloned into both ring + /// pollers; the thread ends when every clone is dropped. + /// + /// `modules` is shared with the attach worker, which snapshots mappings + /// while it holds a process stopped. + pub fn start( + config: StackCaptureConfig, + modules: Arc>, + ) -> (Self, Sender) { + let (sender, receiver) = mpsc::channel(); + let folding = modules.clone(); + let thread = thread::spawn(move || fold_records(receiver, folding)); + ( + Self { + config, + modules, + thread, + }, + sender, + ) + } + + /// Join the folding thread, resolve frame-pointer chains while the BPF maps + /// are still alive, write the dump when configured, and return the report. + pub fn finish( + self, + stats: StackCaptureStats, + resolve_fp: impl Fn(i64) -> Vec, + ) -> Result { + let Self { + config, + modules, + thread, + } = self; + let state = thread + .join() + .map_err(|_| anyhow!("stack recorder folding thread panicked"))?; + let FoldingState { + definitions, + occurrences, + alloc_events_with_stack, + alloc_events_without_stack, + .. + } = state; + + let mut definition_reports = definitions + .into_iter() + .map(|(hash, definition)| StackDefinitionReport { + hash, + stackid: definition.stackid, + sp: definition.sp, + pid: definition.pid, + tid: definition.tid, + truncated: definition.truncated, + copy_len: definition.copy_len as usize, + occurrences: occurrences.get(&hash).copied().unwrap_or(0), + regs: definition.regs.to_vec(), + fp_chain: resolve_fp(definition.stackid), + }) + .collect::>(); + definition_reports.sort_unstable_by(|left, right| { + right + .occurrences + .cmp(&left.occurrences) + .then_with(|| left.hash.cmp(&right.hash)) + }); + + let report = StackReport { + copy_size: config.copy_size, + stats, + alloc_events_with_stack, + alloc_events_without_stack, + unique_stacks: definition_reports.len(), + definitions: definition_reports, + modules: modules.lock().snapshot(), + }; + info!("{}", report.summary()); + + if let Some(path) = config.dump_path { + info!("writing stack report to {}", path.display()); + report.write_json(&path)?; + } + + Ok(report) + } +} + +fn fold_records( + receiver: Receiver, + modules: Arc>, +) -> FoldingState { + let mut state = FoldingState::new(modules); + for record in receiver { + state.record(record); + } + state +} diff --git a/crates/memtrack/src/ebpf/stacks/report.rs b/crates/memtrack/src/ebpf/stacks/report.rs new file mode 100644 index 00000000..08bd90ea --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/report.rs @@ -0,0 +1,82 @@ +use crate::prelude::*; + +use super::counters::StackCaptureStats; +use super::modules::ModuleMapping; + +/// Raw stack bytes are deliberately not serialized: this report measures the +/// payload rather than consuming it, so only `copy_len` survives. +#[derive(Debug, serde::Serialize)] +pub struct StackDefinitionReport { + pub hash: u64, + pub stackid: i64, + pub sp: u64, + pub pid: u32, + pub tid: u32, + pub truncated: bool, + pub copy_len: usize, + pub occurrences: u64, + pub regs: Vec, + pub fp_chain: Vec, +} + +#[derive(Debug, serde::Serialize)] +pub struct StackReport { + pub copy_size: u32, + pub stats: StackCaptureStats, + pub alloc_events_with_stack: u64, + pub alloc_events_without_stack: u64, + pub unique_stacks: usize, + pub definitions: Vec, + pub modules: std::collections::BTreeMap>, +} + +impl StackReport { + pub fn write_json(&self, path: &std::path::Path) -> Result<()> { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating report directory {}", parent.display()))?; + } + } + + let file = std::fs::File::create(path) + .with_context(|| format!("creating stack report {}", path.display()))?; + serde_json::to_writer_pretty(file, self) + .with_context(|| format!("serializing stack report {}", path.display())) + } + + pub fn summary(&self) -> String { + let dedup_ratio = if self.unique_stacks == 0 { + 0.0 + } else { + self.alloc_events_with_stack as f64 / self.unique_stacks as f64 + }; + let mut summary = format!( + "stack copy size={} bytes; unique stacks={}; alloc events with stack={}; alloc events without stack={}; dedup ratio={:.1}; truncated={}", + self.copy_size, + self.unique_stacks, + self.alloc_events_with_stack, + self.alloc_events_without_stack, + dedup_ratio, + self.stats.truncated, + ); + + if self.stats.copy_failed != 0 { + summary.push_str(&format!("; copy_failed={}", self.stats.copy_failed)); + } + if self.stats.hash_map_full != 0 { + summary.push_str(&format!("; hash_map_full={}", self.stats.hash_map_full)); + } + if self.stats.stackid_failed != 0 { + summary.push_str(&format!("; stackid_failed={}", self.stats.stackid_failed)); + } + if self.stats.ring_full != 0 { + summary.push_str(&format!("; ring_full={}", self.stats.ring_full)); + } + if self.stats.preempted != 0 { + summary.push_str(&format!("; preempted={}", self.stats.preempted)); + } + + summary + } +} From 56bb6e64cbf1df5611a74099b6fe49fb184bda75 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:01:07 +0200 Subject: [PATCH 03/16] feat(memtrack): enable stack capture through the tracker Wire the capture rodata and map sizing into skeleton load, poll the stack-definition ring alongside the event ring, and expose the loss counters and frame-pointer chains. The attach worker snapshots module mappings while it holds a process stopped, which is the only point they are guaranteed readable. Guard the lifecycle: finishing with a live session would block forever on the recorder, and a second spawn would leave the capture rings undrained, so both now fail with a descriptive error. With capture disabled the ring buffer and frame-pointer map shrink to the allocator minimum rather than reserving tens of MiB. Refs COD-3222 --- crates/memtrack/src/ebpf/attach_worker.rs | 17 +++- crates/memtrack/src/ebpf/memtrack/maps.rs | 35 ++++++++ crates/memtrack/src/ebpf/memtrack/mod.rs | 62 ++++++++++++- crates/memtrack/src/ebpf/tracker.rs | 104 ++++++++++++++++++++-- crates/memtrack/src/session.rs | 14 +++ 5 files changed, 219 insertions(+), 13 deletions(-) diff --git a/crates/memtrack/src/ebpf/attach_worker.rs b/crates/memtrack/src/ebpf/attach_worker.rs index 61a49c4e..b2abf833 100644 --- a/crates/memtrack/src/ebpf/attach_worker.rs +++ b/crates/memtrack/src/ebpf/attach_worker.rs @@ -12,6 +12,7 @@ use std::thread::JoinHandle; use std::time::Duration; use super::proc_fs::{Resolution, resolve_mapping, wait_all_stopped}; +use super::stacks::modules::ModuleTracker; const STOP_DEADLINE: Duration = Duration::from_secs(1); const POLL_INTERVAL_MS: u64 = 10; @@ -43,7 +44,10 @@ pub(crate) struct AttachWorker { } impl AttachWorker { - pub(crate) fn start(bpf: Arc>) -> Result { + pub(crate) fn start( + bpf: Arc>, + modules: Option>>, + ) -> Result { let shutdown = Arc::new(AtomicBool::new(false)); let fatal = Arc::new(Mutex::new(None)); let root_pid = Arc::new(AtomicI32::new(0)); @@ -58,6 +62,7 @@ impl AttachWorker { shutdown: shutdown.clone(), fatal: fatal.clone(), root_pid: root_pid.clone(), + modules, }; let handle = std::thread::spawn(move || worker.run()); @@ -126,6 +131,7 @@ struct Worker { shutdown: Arc, fatal: Arc>>, root_pid: Arc, + modules: Option>>, } impl Worker { @@ -211,6 +217,15 @@ impl Worker { } } + // Read mappings while the pids are stopped; short-lived processes may + // exit before their captured stacks reach userspace. + if let Some(modules) = &self.modules { + let mut modules = modules.lock(); + for pid in &stopped { + modules.refresh(*pid); + } + } + Ok(()) } diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index c7376d46..939f1bf9 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -1,4 +1,5 @@ use super::MemtrackBpf; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; @@ -68,6 +69,40 @@ impl MemtrackBpf { ) } + pub fn stack_capture_stats(&self) -> Result { + StackCaptureStats::read(with_skel!(self, skel => &skel.maps.stack_counters)) + } + + /// The frame-pointer walk recorded under `stackid`, innermost frame first. + /// Empty when the id is negative (the walk failed) or the entry was evicted. + /// Reading it is best effort: a missing chain costs the fallback for one + /// stack, not the run. + pub fn fp_chain(&self, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let entry = with_skel!(self, skel => skel + .maps + .stack_traces + .lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY)); + let value = match entry { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() + } + pub fn ownership_maps(&self) -> Result { let owner_by_mm = entries(with_skel!(self, skel => &skel.maps.owner_by_mm))?; let mm_by_pid = entries(with_skel!(self, skel => &skel.maps.mm_by_pid))?; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 8586872e..1ef5002d 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -7,6 +7,7 @@ use std::mem::MaybeUninit; use std::path::Path; use crate::ebpf::poller::RingBufferPoller; +use crate::ebpf::stacks::config::StackCaptureConfig; mod token { include!(concat!(env!("OUT_DIR"), "/memtrack_token.skel.rs")); @@ -123,19 +124,25 @@ pub struct MemtrackBpf { impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool) -> Result { + pub fn new_with_rmap(track_rmap: bool, stacks: Option<&StackCaptureConfig>) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap) + Self::with_variant(variant, track_rmap, stacks) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. - pub fn with_variant(variant: BpfVariant, track_rmap: bool) -> Result { + /// + /// `stacks` turns on allocation stack capture and sizes its maps. + pub fn with_variant( + variant: BpfVariant, + track_rmap: bool, + stacks: Option<&StackCaptureConfig>, + ) -> Result { let page_shift = page_shift()?; let rmap = if track_rmap { RmapSupport::detect() @@ -164,6 +171,25 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } + if let Some(stacks) = stacks { + rodata.capture_stacks_enabled = 1; + rodata.stack_copy_size = stacks.copy_size; + } + } + + // Avoid reserving the stack maps when capture is disabled. A + // ring buffer's size must stay a power-of-two page count. + match stacks { + Some(stacks) => open_skel + .maps + .stack_traces + .set_max_entries(stacks.stack_trace_capacity)?, + None => { + open_skel.maps.stack_defs.set_max_entries(4096)?; + open_skel.maps.stack_traces.set_max_entries(1)?; + open_skel.maps.seen_stack_hashes.set_max_entries(1)?; + open_skel.maps.pending_stack_hash.set_max_entries(1)?; + } } // Autoload is decided before load(), so fentries whose targets @@ -214,14 +240,42 @@ impl MemtrackBpf { /// Poll the allocation-event ring buffer into `tx`. The returned poller /// keeps the pipeline alive; events stop flowing when it is dropped. + /// + /// `stack_tx` receives the stack identity carried by each allocation event. + /// It rides along on this ring rather than on its own because the identity + /// lives inside the event record. pub fn poll_events_with_channel( &self, poll_interval_ms: u64, tx: std::sync::mpsc::Sender, + stack_tx: Option>, ) -> Result { + let parse = move |data: &[u8]| { + if let Some(stack_tx) = &stack_tx + && let Some(record) = crate::ebpf::events::parse_alloc_stack_record(data) + { + let _ = stack_tx.send(record); + } + crate::ebpf::events::parse_event(data) + }; + with_skel!(self, skel => RingBufferPoller::new( &skel.maps.events, - crate::ebpf::events::parse_event, + parse, + tx, + poll_interval_ms, + )) + } + + /// Poll the stack-definition ring buffer into `tx`. + pub(crate) fn poll_stack_definitions( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.stack_defs, + crate::ebpf::stacks::events::parse_stack_definition, tx, poll_interval_ms, )) diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 914dfe22..7dc48773 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,5 +1,9 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; +use crate::ebpf::stacks::config::StackCaptureConfig; +use crate::ebpf::stacks::modules::ModuleTracker; +use crate::ebpf::stacks::recorder::StackRecorder; +use crate::ebpf::stacks::report::StackReport; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; @@ -7,27 +11,47 @@ use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc; pub struct Tracker { bpf: Arc>, worker: Mutex>, allocators: bool, + stack_config: Mutex>, + stack_modules: Option>>, + stack_recorder: Mutex>, + stack_report: Mutex>, + live_sessions: Arc, } impl Tracker { /// Create a new tracker. The exec-mapping watcher discovers and attaches /// allocator probes as the tracked process tree maps executable files. pub fn new() -> Result { + Self::with_stack_config(StackCaptureConfig::from_env()) + } + + /// Like [`Tracker::new`], with allocation stack capture forced on. + pub fn new_with_stack_capture(config: StackCaptureConfig) -> Result { + Self::with_stack_config(Some(config)) + } + + fn with_stack_config(stacks: Option) -> Result { let track_rmap = Self::track_rmap_from_env(); - Self::build(MemtrackBpf::new_with_rmap(track_rmap)?, true) + let bpf = MemtrackBpf::new_with_rmap(track_rmap, stacks.as_ref())?; + Self::build(bpf, true, stacks) } /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of /// the detected one. pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = Self::track_rmap_from_env(); - Self::build(MemtrackBpf::with_variant(variant, track_rmap)?, true) + Self::build( + MemtrackBpf::with_variant(variant, track_rmap, None)?, + true, + None, + ) } fn track_rmap_from_env() -> bool { @@ -38,10 +62,14 @@ impl Tracker { /// rmap fentries) without allocator probes; no exec-mapping watcher or /// attach worker runs. pub fn new_without_allocators_with_rmap(track_rmap: bool) -> Result { - Self::build(MemtrackBpf::new_with_rmap(track_rmap)?, false) + Self::build(MemtrackBpf::new_with_rmap(track_rmap, None)?, false, None) } - fn build(mut bpf: MemtrackBpf, allocators: bool) -> Result { + fn build( + mut bpf: MemtrackBpf, + allocators: bool, + stacks: Option, + ) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; @@ -50,8 +78,11 @@ impl Tracker { } let bpf = Arc::new(Mutex::new(bpf)); + let stack_modules = stacks + .as_ref() + .map(|_| Arc::new(Mutex::new(ModuleTracker::new()))); let worker = if allocators { - Some(AttachWorker::start(bpf.clone())?) + Some(AttachWorker::start(bpf.clone(), stack_modules.clone())?) } else { None }; @@ -60,6 +91,11 @@ impl Tracker { bpf, worker: Mutex::new(worker), allocators, + stack_config: Mutex::new(stacks), + stack_modules, + stack_recorder: Mutex::new(None), + stack_report: Mutex::new(None), + live_sessions: Arc::new(AtomicUsize::new(0)), }) } @@ -72,6 +108,9 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { + if self.stack_modules.is_some() && self.stack_config.lock().is_none() { + bail!("stack capture supports a single spawned command per tracker"); + } let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); @@ -86,15 +125,40 @@ impl Tracker { None => {} } + let stack_records = if let Some(config) = self.stack_config.lock().take() { + let Some(modules) = self.stack_modules.clone() else { + bail!("stack capture module tracker is unavailable"); + }; + let (recorder, tx) = StackRecorder::start(config, modules); + *self.stack_recorder.lock() = Some(recorder); + Some(tx) + } else { + None + }; + let (tx, rx) = mpsc::channel(); - let poller = { + let (poller, stack_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; - bpf.poll_events_with_channel(10, tx)? + let stack_poller = match &stack_records { + Some(tx) => Some(bpf.poll_stack_definitions(10, tx.clone())?), + None => None, + }; + ( + bpf.poll_events_with_channel(10, tx, stack_records)?, + stack_poller, + ) }; resume(pid)?; - Ok(Session::new(child, rx, poller)) + self.live_sessions.fetch_add(1, Ordering::SeqCst); + Ok(Session::new( + child, + rx, + poller, + stack_poller, + self.live_sessions.clone(), + )) } /// Enable allocator-event tracking in the BPF program. Lifetime events @@ -122,13 +186,37 @@ impl Tracker { /// Stop the attach worker, if any, and surface any fatal error it recorded, /// including missed exec mappings (incomplete allocator coverage). + /// + /// All [`Session`] values must be dropped before calling this method. It + /// returns an error otherwise because the recorder cannot finish while its + /// ring pollers hold channel senders. pub fn finish(&self) -> Result<()> { + let live_sessions = self.live_sessions.load(Ordering::SeqCst); + if live_sessions != 0 { + bail!( + "{live_sessions} session(s) still alive; drop them first because the recorder cannot finish while ring pollers hold channel senders" + ); + } + if let Some(recorder) = self.stack_recorder.lock().take() { + let bpf = self.bpf.lock(); + let stats = bpf.stack_capture_stats()?; + let report = recorder.finish(stats, |stackid| bpf.fp_chain(stackid))?; + drop(bpf); + *self.stack_report.lock() = Some(report); + } + match self.worker.lock().take() { Some(worker) => worker.finish(), None => Ok(()), } } + /// The stack-capture report produced by [`Self::finish`]. Can only be taken + /// once, and is `None` when capture was off. + pub fn take_stack_report(&self) -> Option { + self.stack_report.lock().take() + } + /// Detach all attached probes. Called explicitly at teardown because the /// process may exit without ever dropping the tracker (the IPC thread holds /// an Arc clone), in which case the kernel would close each link fd serially. diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 7aec33fe..91522c36 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -2,6 +2,8 @@ use crate::ebpf::poller::RingBufferPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; use std::process::{Child, ExitStatus}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::Receiver; /// A spawned, tracked process together with its event pipeline. The pipeline @@ -10,6 +12,8 @@ pub struct Session { child: Child, events: Option>, _poller: RingBufferPoller, + _stack_poller: Option, + live_sessions: Arc, } impl Session { @@ -17,11 +21,15 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, + stack_poller: Option, + live_sessions: Arc, ) -> Self { Self { child, events: Some(events), _poller: poller, + _stack_poller: stack_poller, + live_sessions, } } @@ -39,3 +47,9 @@ impl Session { Ok(self.child.wait()?) } } + +impl Drop for Session { + fn drop(&mut self) { + self.live_sessions.fetch_sub(1, Ordering::SeqCst); + } +} From ac144872f703642003f713ccfcc15ff7a9917559 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 15:01:18 +0200 Subject: [PATCH 04/16] test(memtrack): cover allocation stack capture Add a fixture with two non-inlinable malloc call paths and privileged tests over it: distinct call paths get distinct identities with module mappings for the binary and libc, repeated calls deduplicate, and the default-off path still reports allocations. Two cases guard failure modes the default budget cannot reach. The maximum copy budget is the only configuration that exercises the verifier's instruction limit, since the frozen rodata makes the copy and hash loops scale with the configured size. Shrinking the frame-pointer map to one slot proves exhaustion costs only the fallback chain, never an allocation event. Refs COD-3222 --- crates/memtrack/testdata/stack_paths.c | 46 ++++ crates/memtrack/tests/shared.rs | 17 ++ crates/memtrack/tests/stack_tests.rs | 336 +++++++++++++++++++++++++ 3 files changed, 399 insertions(+) create mode 100644 crates/memtrack/testdata/stack_paths.c create mode 100644 crates/memtrack/tests/stack_tests.rs diff --git a/crates/memtrack/testdata/stack_paths.c b/crates/memtrack/testdata/stack_paths.c new file mode 100644 index 00000000..0cee7f43 --- /dev/null +++ b/crates/memtrack/testdata/stack_paths.c @@ -0,0 +1,46 @@ +#include +#include + +static volatile void *escaped_pointer; +static volatile unsigned int remaining_a = 50; +static volatile unsigned int remaining_b = 50; + +__attribute__((noinline)) static void path_a_inner(void) { + void *pointer = malloc(64); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_a(void) { + while (remaining_a != 0) { + path_a_inner(); + --remaining_a; + } +} + +__attribute__((noinline)) static void path_b_inner(void) { + void *pointer = malloc(192); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_b(void) { + while (remaining_b != 0) { + path_b_inner(); + --remaining_b; + } +} + +int main(void) { + void *marker_before = malloc(0xC0D59EED); + escaped_pointer = marker_before; + free(marker_before); + + path_a(); + path_b(); + + void *marker_after = malloc(0xC0D59EED); + escaped_pointer = marker_after; + free(marker_after); + return 0; +} diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 499d74c5..16d9fad1 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -221,6 +221,23 @@ pub fn track_command_with_rmap_maps( let maps = tracker.ownership_maps()?; Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } +/// Track a command with allocation stack capture enabled, returning its events +/// and stack report. +pub fn track_command_with_stacks( + command: Command, + config: memtrack::StackCaptureConfig, +) -> anyhow::Result<( + Vec, + memtrack::StackReport, + std::thread::JoinHandle<()>, +)> { + let tracker = Tracker::new_with_stack_capture(config)?; + let (tracker, events, ()) = run_tracked(command, tracker, |_, _| Ok(()))?; + let report = tracker + .take_stack_report() + .context("tracker did not produce a stack report")?; + Ok((events, report, std::thread::spawn(move || drop(tracker)))) +} /// Track a command with rmap hooks and snapshot the ownership maps at a /// fixture-signalled checkpoint. diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs new file mode 100644 index 00000000..7e1fe24d --- /dev/null +++ b/crates/memtrack/tests/stack_tests.rs @@ -0,0 +1,336 @@ +#[macro_use] +mod shared; + +use std::collections::HashSet; +use std::path::Path; +use std::process::Command; +use tempfile::TempDir; +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + "stack_paths", + temp_dir.path(), + )?; + let dump_path = temp_dir.path().join("stacks.json"); + let copy_size = memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE; + let (events, report, thread_handle) = shared::track_command_with_stacks( + Command::new(&binary), + memtrack::StackCaptureConfig { + dump_path: Some(dump_path.clone()), + ..memtrack::StackCaptureConfig::with_copy_size(copy_size) + }, + )?; + + assert!( + report.definitions.len() >= 2, + "expected at least two stack definitions, got {} ({} events)", + report.definitions.len(), + events.len() + ); + let hashes: HashSet<_> = report + .definitions + .iter() + .map(|definition| definition.hash) + .collect(); + assert_eq!( + hashes.len(), + report.definitions.len(), + "stack definitions must be deduplicated by unique hash" + ); + assert!( + report + .definitions + .iter() + .all(|definition| definition.copy_len > 0), + "every stack definition must contain copied stack bytes" + ); + assert!( + report + .definitions + .iter() + .all(|definition| definition.sp != 0), + "every stack definition must include a nonzero stack pointer" + ); + // `truncated` means the budget ran out, not that the copy was short: a copy + // that stops at the top of the stack mapping is complete however short it is. + // The kernel only ever copies whole chunks, so the emitted length is exactly + // the hashed length. + // + // Whether any capture is complete at this budget depends on how much stack + // is readable above the stack pointer, which shifts with the environment + // size, so only the maximum budget can guarantee completeness. + assert!( + report.definitions.iter().all(|definition| { + definition.copy_len <= copy_size as usize + && definition.copy_len % 512 == 0 + && definition.truncated == (definition.copy_len == copy_size as usize) + }), + "stack copies must be whole chunks within the budget and flagged truncated only when they fill it: {:?}", + report + .definitions + .iter() + .map(|definition| (definition.copy_len, definition.truncated)) + .collect::>() + ); + assert_eq!( + report.stats.copy_failed, 0, + "stack capture reported {} copy failures", + report.stats.copy_failed + ); + assert_eq!( + report.stats.ring_full, 0, + "stack definition ring buffer reported {} full events", + report.stats.ring_full + ); + assert!( + report.alloc_events_with_stack > 0, + "expected allocation events carrying captured stack hashes" + ); + + // Every chain starts inside the allocator the probe sits on, so the paths + // only diverge from the second frame onwards. + let all_stackids_negative = report + .definitions + .iter() + .all(|definition| definition.stackid < 0); + if all_stackids_negative { + assert!( + report + .definitions + .iter() + .all(|definition| definition.fp_chain.is_empty()), + "all stack IDs are negative, so every FP chain must be empty" + ); + } else { + let chains: HashSet<_> = report + .definitions + .iter() + .map(|definition| definition.fp_chain.clone()) + .collect(); + assert!( + chains.len() >= 2, + "some stack IDs are available, but the {} definitions produced only {} distinct frame-pointer chains: {:?}", + report.definitions.len(), + chains.len(), + chains + ); + } + + let fixture_pid = report + .definitions + .first() + .expect("at least one definition was asserted above") + .pid; + let modules = report + .modules + .get(&fixture_pid) + .expect("stack report must include module mappings for the fixture pid"); + let fixture_name = binary + .file_name() + .expect("compiled fixture must have a file name") + .to_string_lossy() + .into_owned(); + assert!( + modules.iter().any(|mapping| { + Path::new(&mapping.path) + .file_name() + .is_some_and(|name| name.to_string_lossy() == fixture_name.as_str()) + }), + "module mappings for pid {fixture_pid} must include fixture binary {fixture_name:?}" + ); + assert!( + modules.iter().any(|mapping| { + Path::new(&mapping.path) + .file_name() + .is_some_and(|name| name.to_string_lossy().starts_with("libc")) + }), + "module mappings for pid {fixture_pid} must include a libc image" + ); + + let dumped: serde_json::Value = serde_json::from_reader(std::fs::File::open(&dump_path)?) + .map_err(|error| { + format!( + "stack dump at {} did not parse: {error}", + dump_path.display() + ) + })?; + assert_eq!( + dumped["unique_stacks"].as_u64(), + Some(report.unique_stacks as u64), + "the written dump must agree with the in-memory report" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn dedup_collapses_repeated_call_paths() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + "stack_paths_dedup", + temp_dir.path(), + )?; + let (events, report, thread_handle) = shared::track_command_with_stacks( + Command::new(&binary), + memtrack::StackCaptureConfig::with_copy_size( + memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE, + ), + )?; + + assert!( + report.alloc_events_with_stack > report.unique_stacks as u64, + "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", + report.alloc_events_with_stack, + report.unique_stacks, + events.len() + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// The largest budget stresses the verifier hardest: the copy loop and its +/// unrolled per-chunk hash both scale with the configured size, so a program +/// that loads at the default can still exceed the instruction limit here. +/// It is also the only budget at which nothing can be budget-limited, because +/// the stack mapping always ends first. +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn max_copy_budget_loads_and_captures_whole_stacks() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + "stack_paths_max", + temp_dir.path(), + )?; + // Clamped down to the largest copy a stack-definition record can carry. + let (_events, report, thread_handle) = shared::track_command_with_stacks( + Command::new(&binary), + memtrack::StackCaptureConfig::with_copy_size(u32::MAX), + )?; + + assert!( + report.copy_size > memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE, + "the clamp must land above the default budget, got {}", + report.copy_size + ); + assert!( + !report.definitions.is_empty(), + "expected stack definitions at the maximum copy budget" + ); + assert!( + report + .definitions + .iter() + .all(|definition| !definition.truncated), + "no capture can be budget-limited at the maximum budget: {:?}", + report + .definitions + .iter() + .map(|definition| (definition.copy_len, definition.truncated)) + .collect::>() + ); + assert_eq!( + report.stats.copy_failed, 0, + "stack capture reported {} copy failures at the maximum budget", + report.stats.copy_failed + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// Exhausting the frame-pointer walk slots must degrade the fallback chain and +/// nothing else: every allocation still reports its stack identity. +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn exhausted_stack_trace_map_costs_only_fp_chains() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + "stack_paths_tiny_map", + temp_dir.path(), + )?; + + let baseline = shared::track_command_with_stacks( + Command::new(&binary), + memtrack::StackCaptureConfig::with_copy_size( + memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE, + ), + )?; + let (_, baseline_report, baseline_handle) = baseline; + baseline_handle + .join() + .expect("tracker teardown thread panicked"); + + let (_events, report, thread_handle) = shared::track_command_with_stacks( + Command::new(&binary), + memtrack::StackCaptureConfig { + stack_trace_capacity: 1, + ..memtrack::StackCaptureConfig::with_copy_size( + memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE, + ) + }, + )?; + + assert!( + report.stats.stackid_failed > 0, + "a one-slot frame-pointer map must report failed walks, got {}", + report.stats.stackid_failed + ); + assert_eq!( + report.alloc_events_with_stack, baseline_report.alloc_events_with_stack, + "allocation totals must be unaffected by frame-pointer map exhaustion" + ); + assert_eq!( + report.alloc_events_without_stack, 0, + "no allocation may lose its stack identity when only the walk fails" + ); + assert_eq!( + report.unique_stacks, baseline_report.unique_stacks, + "stack identity comes from the copied bytes, not the frame-pointer walk" + ); + assert_eq!( + report.stats.copy_failed, 0, + "stack copies must still succeed with an exhausted frame-pointer map" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn default_off_leaves_allocation_events_intact() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + "stack_paths_disabled", + temp_dir.path(), + )?; + let (events, thread_handle) = shared::track_binary(&binary)?; + + assert!( + !events.is_empty(), + "default-off tracking must still report allocation events" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} From 2012585614529949d85d44fe8c0163738376cf69 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 16:12:45 +0200 Subject: [PATCH 05/16] fixup! feat(memtrack): capture allocation stacks in eBPF --- crates/memtrack/src/ebpf/c/event.h | 5 +++-- crates/memtrack/src/ebpf/c/stack_capture.bpf.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 8401d518..bc0bc43b 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -46,8 +46,9 @@ struct stack_regs { * from `sp` follow it. */ struct stack_def_header { uint64_t hash; - int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ - uint64_t sp; /* user stack pointer the copy starts at */ + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ + uint64_t sp; /* user stack pointer the copy starts at */ uint32_t pid; uint32_t tid; uint32_t copy_len; diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h index d4168b39..090fd6b1 100644 --- a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -202,6 +202,7 @@ static __always_inline void capture_stack_inner(struct pt_regs* ctx, struct task } scratch->header.hash = hash; + scratch->header.timestamp = bpf_ktime_get_ns(); scratch->header.stackid = stackid; scratch->header.sp = sp; scratch->header.pid = ids.tgid; From 9d5ed9c91957397f78274fe37e33744016785657 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 16:12:59 +0200 Subject: [PATCH 06/16] fixup! feat(memtrack): add userspace stack-capture module --- Cargo.lock | 11 + crates/memtrack/src/ebpf/events.rs | 45 +--- crates/memtrack/src/ebpf/mod.rs | 1 - crates/memtrack/src/ebpf/stacks/config.rs | 6 - crates/memtrack/src/ebpf/stacks/events.rs | 85 +++--- crates/memtrack/src/ebpf/stacks/mod.rs | 3 - crates/memtrack/src/ebpf/stacks/modules.rs | 165 ------------ crates/memtrack/src/ebpf/stacks/recorder.rs | 157 ----------- crates/memtrack/src/ebpf/stacks/report.rs | 82 ------ crates/runner-shared/Cargo.toml | 1 + .../src/artifacts/memtrack/mod.rs | 253 +++++++++++++++++- .../src/artifacts/memtrack/pipeline.rs | 5 +- 12 files changed, 317 insertions(+), 497 deletions(-) delete mode 100644 crates/memtrack/src/ebpf/stacks/modules.rs delete mode 100644 crates/memtrack/src/ebpf/stacks/recorder.rs delete mode 100644 crates/memtrack/src/ebpf/stacks/report.rs diff --git a/Cargo.lock b/Cargo.lock index 9cf2031c..1f63a302 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3615,6 +3615,7 @@ dependencies = [ "rmp", "rmp-serde", "serde", + "serde_bytes", "serde_json", "zstd", ] @@ -4061,6 +4062,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index a3c76deb..9fa37561 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -34,6 +34,7 @@ pub fn parse_event(data: &[u8]) -> Option { event.data.alloc.addr, MemtrackEventKind::Malloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_FREE => (event.data.free.addr, MemtrackEventKind::Free), @@ -41,6 +42,7 @@ pub fn parse_event(data: &[u8]) -> Option { event.data.alloc.addr, MemtrackEventKind::Calloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_REALLOC => ( @@ -48,12 +50,14 @@ pub fn parse_event(data: &[u8]) -> Option { MemtrackEventKind::Realloc { old_addr: Some(event.data.realloc.old_addr), size: event.data.realloc.size, + stack_hash: event.data.realloc.stack_hash, }, ), EVENT_TYPE_ALIGNED_ALLOC => ( event.data.alloc.addr, MemtrackEventKind::AlignedAlloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_MMAP => ( @@ -110,35 +114,6 @@ pub fn parse_event(data: &[u8]) -> Option { kind, }) } -/// The stack identity an allocation event carries, or `None` for event types -/// that have no allocation-site stack. -pub fn parse_alloc_stack_record(data: &[u8]) -> Option { - if data.len() < std::mem::size_of::() { - return None; - } - - // SAFETY: The data must be a valid `bindings::event`. - let event = unsafe { &*(data.as_ptr() as *const bindings::event) }; - - // SAFETY: The fields must be properly initialized in eBPF. - Some(unsafe { - match event.header.event_type as u32 { - EVENT_TYPE_MALLOC => crate::ebpf::stacks::events::StackRecord::Alloc { - hash: event.data.alloc.stack_hash, - }, - EVENT_TYPE_CALLOC => crate::ebpf::stacks::events::StackRecord::Alloc { - hash: event.data.alloc.stack_hash, - }, - EVENT_TYPE_ALIGNED_ALLOC => crate::ebpf::stacks::events::StackRecord::Alloc { - hash: event.data.alloc.stack_hash, - }, - EVENT_TYPE_REALLOC => crate::ebpf::stacks::events::StackRecord::Alloc { - hash: event.data.realloc.stack_hash, - }, - _ => return None, - } - }) -} /// A request from the exec-mapping watcher to attach allocator probes. #[derive(Debug, Clone, Copy)] @@ -186,6 +161,7 @@ mod tests { event.data.realloc.old_addr = 0x1000; event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; + event.data.realloc.stack_hash = 0xbeef; let bytes = event_bytes(&event); @@ -197,9 +173,14 @@ mod tests { assert_eq!(parsed.addr, 0x2000); match parsed.kind { - MemtrackEventKind::Realloc { old_addr, size } => { + MemtrackEventKind::Realloc { + old_addr, + size, + stack_hash, + } => { assert_eq!(old_addr, Some(0x1000)); assert_eq!(size, 256); + assert_eq!(stack_hash, 0xbeef); } _ => panic!("Expected Realloc event kind"), } @@ -215,6 +196,7 @@ mod tests { event.header.tid = 2000; event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; + event.data.alloc.stack_hash = 0x1234; let bytes = event_bytes(&event); @@ -226,8 +208,9 @@ mod tests { assert_eq!(parsed.addr, 0x1000); match parsed.kind { - MemtrackEventKind::Malloc { size } => { + MemtrackEventKind::Malloc { size, stack_hash } => { assert_eq!(size, 128); + assert_eq!(stack_hash, 0x1234); } _ => panic!("Expected Malloc event kind"), } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index aa3ceb74..e580276f 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -12,5 +12,4 @@ pub use memtrack::{ }; pub use stacks::config::StackCaptureConfig; pub use stacks::counters::StackCaptureStats; -pub use stacks::report::{StackDefinitionReport, StackReport}; pub use tracker::Tracker; diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs index d5a757af..f7ed299d 100644 --- a/crates/memtrack/src/ebpf/stacks/config.rs +++ b/crates/memtrack/src/ebpf/stacks/config.rs @@ -5,7 +5,6 @@ pub struct StackCaptureConfig { /// Frame-pointer walk slots. Exhausting them costs the fallback chain for /// stacks beyond the limit, never an allocation event. pub stack_trace_capacity: u32, - pub dump_path: Option, } impl StackCaptureConfig { @@ -38,13 +37,9 @@ impl StackCaptureConfig { Self::DEFAULT_COPY_SIZE } }; - let dump_path = - std::env::var_os("CODSPEED_MEMTRACK_STACK_DUMP").map(std::path::PathBuf::from); - Some(Self { copy_size: clamp_copy_size(copy_size), stack_trace_capacity: Self::DEFAULT_STACK_TRACE_CAPACITY, - dump_path, }) } @@ -52,7 +47,6 @@ impl StackCaptureConfig { Self { copy_size: clamp_copy_size(copy_size), stack_trace_capacity: Self::DEFAULT_STACK_TRACE_CAPACITY, - dump_path: None, } } } diff --git a/crates/memtrack/src/ebpf/stacks/events.rs b/crates/memtrack/src/ebpf/stacks/events.rs index ac5f4d4a..58125636 100644 --- a/crates/memtrack/src/ebpf/stacks/events.rs +++ b/crates/memtrack/src/ebpf/stacks/events.rs @@ -1,25 +1,7 @@ use crate::prelude::*; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; -#[derive(Debug, Clone)] -pub struct StackDefinition { - pub hash: u64, - pub stackid: i64, - pub sp: u64, - pub pid: u32, - pub tid: u32, - pub truncated: bool, - pub regs: [u64; 33], - /// The payload is validated and measured but not retained. - pub copy_len: u32, -} - -#[derive(Debug)] -pub enum StackRecord { - Definition(Box), - Alloc { hash: u64 }, -} - -pub fn parse_stack_definition(data: &[u8]) -> Option { +pub fn parse_stack_definition(data: &[u8]) -> Option<(MemtrackEvent, i64)> { let header_len = std::mem::size_of::(); if data.len() < header_len { warn!( @@ -51,17 +33,23 @@ pub fn parse_stack_definition(data: &[u8]) -> Option { ); return None; } + let bytes = data[header_len..record_len].to_vec(); + let event = MemtrackEvent { + pid: header.pid as i32, + tid: header.tid as i32, + timestamp: header.timestamp, + addr: 0, + kind: MemtrackEventKind::StackDefinition { + hash: header.hash, + sp: header.sp, + regs: header.regs.reg.to_vec(), + bytes, + fp_chain: Vec::new(), + truncated: header.truncated != 0, + }, + }; - Some(StackRecord::Definition(Box::new(StackDefinition { - hash: header.hash, - stackid: header.stackid, - sp: header.sp, - pid: header.pid, - tid: header.tid, - truncated: header.truncated != 0, - regs: header.regs.reg, - copy_len: header.copy_len, - }))) + Some((event, header.stackid)) } #[cfg(test)] @@ -87,6 +75,7 @@ mod tests { let regs = std::array::from_fn(|index| 0x1000 + index as u64); let header = bindings::stack_def_header { hash: 0x0123_4567_89ab_cdef, + timestamp: 987_654_321, stackid: -17, sp: 0x7fff_1234_5000, pid: 41, @@ -98,21 +87,32 @@ mod tests { }; let payload = [1, 2, 3, 4, 5]; - let record = parse_stack_definition(&encode(header, &payload)); - let Some(StackRecord::Definition(definition)) = record else { - panic!("expected stack definition"); + let (event, stackid) = parse_stack_definition(&encode(header, &payload)).unwrap(); + assert_eq!(event.pid, 41); + assert_eq!(event.tid, 42); + assert_eq!(event.timestamp, 987_654_321); + assert_eq!(event.addr, 0); + assert_eq!(stackid, -17); + + let MemtrackEventKind::StackDefinition { + hash, + sp, + regs: parsed_regs, + bytes, + fp_chain, + truncated, + } = event.kind + else { + panic!("expected StackDefinition event"); }; - assert_eq!(definition.hash, header.hash); - assert_eq!(definition.stackid, header.stackid); - assert_eq!(definition.sp, header.sp); - assert_eq!(definition.pid, header.pid); - assert_eq!(definition.tid, header.tid); - assert!(definition.truncated); - assert_eq!(definition.regs, regs); - assert_eq!(definition.copy_len, payload.len() as u32); + assert_eq!(hash, header.hash); + assert_eq!(sp, header.sp); + assert_eq!(parsed_regs, regs.to_vec()); + assert_eq!(bytes, payload); + assert!(fp_chain.is_empty()); + assert!(truncated); } - #[test] fn truncated_buffer_returns_none() { let data = vec![0; std::mem::size_of::() - 1]; @@ -123,6 +123,7 @@ mod tests { fn missing_payload_returns_none() { let header = bindings::stack_def_header { hash: 1, + timestamp: 100, stackid: 2, sp: 3, pid: 4, diff --git a/crates/memtrack/src/ebpf/stacks/mod.rs b/crates/memtrack/src/ebpf/stacks/mod.rs index f41dc8d9..fc5eee33 100644 --- a/crates/memtrack/src/ebpf/stacks/mod.rs +++ b/crates/memtrack/src/ebpf/stacks/mod.rs @@ -1,6 +1,3 @@ pub mod config; pub mod counters; pub mod events; -pub mod modules; -pub mod recorder; -pub mod report; diff --git a/crates/memtrack/src/ebpf/stacks/modules.rs b/crates/memtrack/src/ebpf/stacks/modules.rs deleted file mode 100644 index 3559509a..00000000 --- a/crates/memtrack/src/ebpf/stacks/modules.rs +++ /dev/null @@ -1,165 +0,0 @@ -use std::collections::{BTreeMap, HashMap}; -use std::time::{Duration, Instant}; - -const RESCAN_INTERVAL: Duration = Duration::from_millis(250); - -#[derive(Debug, Clone, serde::Serialize)] -pub struct ModuleMapping { - pub start: u64, - pub end: u64, - pub offset: u64, - pub path: String, -} - -pub struct ModuleTracker { - mappings: HashMap>, - last_read: HashMap, -} - -impl ModuleTracker { - pub fn new() -> Self { - Self { - mappings: HashMap::new(), - last_read: HashMap::new(), - } - } - - /// Re-read the pid's mappings at most once per [`RESCAN_INTERVAL`]. Callers - /// on the allocation path use this: a process can `dlopen` at any time, but - /// re-reading `/proc` per captured stack would cost more than it recovers. - pub fn observe(&mut self, pid: u32) { - if self - .last_read - .get(&pid) - .is_some_and(|last| last.elapsed() < RESCAN_INTERVAL) - { - return; - } - self.refresh(pid); - } - - /// Re-read the pid's mappings now. Callers holding the process stopped use - /// this: it is the only moment the mappings are guaranteed readable, and a - /// short-lived process is usually gone by the time its stacks surface. - pub fn refresh(&mut self, pid: u32) { - self.last_read.insert(pid, Instant::now()); - - let Ok(contents) = std::fs::read_to_string(format!("/proc/{pid}/maps")) else { - return; - }; - let by_start = self.mappings.entry(pid).or_default(); - for mapping in parse_maps(&contents) { - by_start.insert(mapping.start, mapping); - } - } - - pub fn snapshot(&self) -> BTreeMap> { - self.mappings - .iter() - .map(|(pid, mappings)| (*pid, mappings.values().cloned().collect())) - .collect() - } -} - -fn parse_maps(contents: &str) -> Vec { - contents.lines().filter_map(parse_map_line).collect() -} - -fn parse_map_line(line: &str) -> Option { - let mut fields = line.split_whitespace(); - let range = fields.next()?; - let permissions = fields.next()?; - let offset = fields.next()?; - let device = fields.next()?; - let inode = fields.next()?; - - if !permissions.contains('x') { - return None; - } - let (start, end) = range.split_once('-')?; - let start = u64::from_str_radix(start, 16).ok()?; - let end = u64::from_str_radix(end, 16).ok()?; - if start >= end { - return None; - } - let offset = u64::from_str_radix(offset, 16).ok()?; - parse_device(device)?; - inode.parse::().ok()?; - - let mut remaining = line; - for _ in 0..5 { - let trimmed = remaining.trim_start(); - let end = trimmed.find(|character: char| character.is_whitespace())?; - remaining = &trimmed[end..]; - } - let path = remaining.trim(); - if !path.starts_with('/') { - return None; - } - - Some(ModuleMapping { - start, - end, - offset, - path: path.to_owned(), - }) -} - -fn parse_device(device: &str) -> Option<(u64, u64)> { - let (major, minor) = device.split_once(':')?; - Some(( - u64::from_str_radix(major, 16).ok()?, - u64::from_str_radix(minor, 16).ok()?, - )) -} - -#[cfg(test)] -mod tests { - use super::parse_maps; - - #[test] - fn keeps_executable_file_mapping() { - let mappings = - parse_maps("55a1b2c00000-55a1b2c21000 r-xp 00001000 08:02 1234 /usr/bin/foo\n"); - assert_eq!(mappings.len(), 1); - assert_eq!(mappings[0].start, 0x55a1b2c00000); - assert_eq!(mappings[0].end, 0x55a1b2c21000); - assert_eq!(mappings[0].offset, 0x1000); - assert_eq!(mappings[0].path, "/usr/bin/foo"); - } - - #[test] - fn drops_non_executable_mapping() { - assert!( - parse_maps("55a1b2c00000-55a1b2c21000 r--p 00000000 08:02 1234 /usr/bin/foo\n") - .is_empty() - ); - } - - #[test] - fn drops_pseudo_and_anonymous_mappings() { - let contents = concat!( - "7f0000000000-7f0000001000 r-xp 00000000 00:00 0 [vdso]\n", - "00600000-00601000 r-xp 00000000 00:00 0 [heap]\n", - "7f0000010000-7f0000020000 r-xp 00000000 00:00 0\n", - ); - assert!(parse_maps(contents).is_empty()); - } - - #[test] - fn preserves_spaces_in_path() { - let mappings = - parse_maps("55a1b2c00000-55a1b2c21000 r-xp 00000000 08:02 1234 /opt/my program/bin\n"); - assert_eq!(mappings.len(), 1); - assert_eq!(mappings[0].path, "/opt/my program/bin"); - } - - #[test] - fn skips_malformed_lines() { - let contents = concat!( - "not a maps line\n", - "55a1b2c00000-55a1b2c21000 r-xp not-hex 08:02 1234 /usr/bin/foo\n", - ); - assert!(parse_maps(contents).is_empty()); - } -} diff --git a/crates/memtrack/src/ebpf/stacks/recorder.rs b/crates/memtrack/src/ebpf/stacks/recorder.rs deleted file mode 100644 index 76b27c36..00000000 --- a/crates/memtrack/src/ebpf/stacks/recorder.rs +++ /dev/null @@ -1,157 +0,0 @@ -use crate::prelude::*; - -use parking_lot::Mutex; -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::mpsc::{self, Receiver, Sender}; -use std::thread::{self, JoinHandle}; - -use super::config::StackCaptureConfig; -use super::counters::StackCaptureStats; -use super::events::{StackDefinition, StackRecord}; -use super::modules::ModuleTracker; -use super::report::{StackDefinitionReport, StackReport}; - -pub struct StackRecorder { - config: StackCaptureConfig, - modules: Arc>, - thread: JoinHandle, -} - -struct FoldingState { - definitions: HashMap, - occurrences: HashMap, - alloc_events_with_stack: u64, - alloc_events_without_stack: u64, - modules: Arc>, -} - -impl FoldingState { - fn new(modules: Arc>) -> Self { - Self { - definitions: HashMap::new(), - occurrences: HashMap::new(), - alloc_events_with_stack: 0, - alloc_events_without_stack: 0, - modules, - } - } - - fn record(&mut self, record: StackRecord) { - match record { - StackRecord::Definition(definition) => { - let definition = *definition; - self.modules.lock().observe(definition.pid); - self.definitions - .entry(definition.hash) - .or_insert(definition); - } - StackRecord::Alloc { hash: 0 } => { - self.alloc_events_without_stack += 1; - } - StackRecord::Alloc { hash } => { - self.alloc_events_with_stack += 1; - *self.occurrences.entry(hash).or_default() += 1; - } - } - } -} - -impl StackRecorder { - /// Spawns the folding thread. The returned sender is cloned into both ring - /// pollers; the thread ends when every clone is dropped. - /// - /// `modules` is shared with the attach worker, which snapshots mappings - /// while it holds a process stopped. - pub fn start( - config: StackCaptureConfig, - modules: Arc>, - ) -> (Self, Sender) { - let (sender, receiver) = mpsc::channel(); - let folding = modules.clone(); - let thread = thread::spawn(move || fold_records(receiver, folding)); - ( - Self { - config, - modules, - thread, - }, - sender, - ) - } - - /// Join the folding thread, resolve frame-pointer chains while the BPF maps - /// are still alive, write the dump when configured, and return the report. - pub fn finish( - self, - stats: StackCaptureStats, - resolve_fp: impl Fn(i64) -> Vec, - ) -> Result { - let Self { - config, - modules, - thread, - } = self; - let state = thread - .join() - .map_err(|_| anyhow!("stack recorder folding thread panicked"))?; - let FoldingState { - definitions, - occurrences, - alloc_events_with_stack, - alloc_events_without_stack, - .. - } = state; - - let mut definition_reports = definitions - .into_iter() - .map(|(hash, definition)| StackDefinitionReport { - hash, - stackid: definition.stackid, - sp: definition.sp, - pid: definition.pid, - tid: definition.tid, - truncated: definition.truncated, - copy_len: definition.copy_len as usize, - occurrences: occurrences.get(&hash).copied().unwrap_or(0), - regs: definition.regs.to_vec(), - fp_chain: resolve_fp(definition.stackid), - }) - .collect::>(); - definition_reports.sort_unstable_by(|left, right| { - right - .occurrences - .cmp(&left.occurrences) - .then_with(|| left.hash.cmp(&right.hash)) - }); - - let report = StackReport { - copy_size: config.copy_size, - stats, - alloc_events_with_stack, - alloc_events_without_stack, - unique_stacks: definition_reports.len(), - definitions: definition_reports, - modules: modules.lock().snapshot(), - }; - info!("{}", report.summary()); - - if let Some(path) = config.dump_path { - info!("writing stack report to {}", path.display()); - report.write_json(&path)?; - } - - Ok(report) - } -} - -fn fold_records( - receiver: Receiver, - modules: Arc>, -) -> FoldingState { - let mut state = FoldingState::new(modules); - for record in receiver { - state.record(record); - } - state -} diff --git a/crates/memtrack/src/ebpf/stacks/report.rs b/crates/memtrack/src/ebpf/stacks/report.rs deleted file mode 100644 index 08bd90ea..00000000 --- a/crates/memtrack/src/ebpf/stacks/report.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::prelude::*; - -use super::counters::StackCaptureStats; -use super::modules::ModuleMapping; - -/// Raw stack bytes are deliberately not serialized: this report measures the -/// payload rather than consuming it, so only `copy_len` survives. -#[derive(Debug, serde::Serialize)] -pub struct StackDefinitionReport { - pub hash: u64, - pub stackid: i64, - pub sp: u64, - pub pid: u32, - pub tid: u32, - pub truncated: bool, - pub copy_len: usize, - pub occurrences: u64, - pub regs: Vec, - pub fp_chain: Vec, -} - -#[derive(Debug, serde::Serialize)] -pub struct StackReport { - pub copy_size: u32, - pub stats: StackCaptureStats, - pub alloc_events_with_stack: u64, - pub alloc_events_without_stack: u64, - pub unique_stacks: usize, - pub definitions: Vec, - pub modules: std::collections::BTreeMap>, -} - -impl StackReport { - pub fn write_json(&self, path: &std::path::Path) -> Result<()> { - if let Some(parent) = path.parent() { - if !parent.as_os_str().is_empty() { - std::fs::create_dir_all(parent) - .with_context(|| format!("creating report directory {}", parent.display()))?; - } - } - - let file = std::fs::File::create(path) - .with_context(|| format!("creating stack report {}", path.display()))?; - serde_json::to_writer_pretty(file, self) - .with_context(|| format!("serializing stack report {}", path.display())) - } - - pub fn summary(&self) -> String { - let dedup_ratio = if self.unique_stacks == 0 { - 0.0 - } else { - self.alloc_events_with_stack as f64 / self.unique_stacks as f64 - }; - let mut summary = format!( - "stack copy size={} bytes; unique stacks={}; alloc events with stack={}; alloc events without stack={}; dedup ratio={:.1}; truncated={}", - self.copy_size, - self.unique_stacks, - self.alloc_events_with_stack, - self.alloc_events_without_stack, - dedup_ratio, - self.stats.truncated, - ); - - if self.stats.copy_failed != 0 { - summary.push_str(&format!("; copy_failed={}", self.stats.copy_failed)); - } - if self.stats.hash_map_full != 0 { - summary.push_str(&format!("; hash_map_full={}", self.stats.hash_map_full)); - } - if self.stats.stackid_failed != 0 { - summary.push_str(&format!("; stackid_failed={}", self.stats.stackid_failed)); - } - if self.stats.ring_full != 0 { - summary.push_str(&format!("; ring_full={}", self.stats.ring_full)); - } - if self.stats.preempted != 0 { - summary.push_str(&format!("; preempted={}", self.stats.preempted)); - } - - summary - } -} diff --git a/crates/runner-shared/Cargo.toml b/crates/runner-shared/Cargo.toml index 8b8f6ab9..9c3c9f18 100644 --- a/crates/runner-shared/Cargo.toml +++ b/crates/runner-shared/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" [dependencies] anyhow = { workspace = true } serde = { workspace = true } +serde_bytes = "0.11" serde_json = { workspace = true } # Pinned to 1.x: 2.0 changes the wire format and serde integration bincode = "1.3" diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index b082a7c6..0778bd76 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -41,7 +41,7 @@ impl MemtrackArtifact { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct MemtrackEvent { pub pid: pid_t, pub tid: pid_t, @@ -51,23 +51,31 @@ pub struct MemtrackEvent { pub kind: MemtrackEventKind, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] pub enum MemtrackEventKind { Malloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Free, Realloc { #[serde(default, skip_serializing_if = "Option::is_none")] old_addr: Option, size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Calloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, AlignedAlloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Mmap { size: u64, @@ -91,8 +99,43 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + + /// Raw stack for one distinct `stack_hash`, emitted once per hash. Allocation events + /// reference it by hash. Unwinding happens off-box. + StackDefinition { + hash: u64, + /// User stack pointer the copy starts at. + sp: u64, + /// Registers by DWARF number for the capturing architecture; 33 entries on x86_64. + regs: Vec, + /// Raw stack bytes read upward from `sp`. + #[serde(with = "serde_bytes")] + bytes: Vec, + /// In-kernel frame-pointer walk, innermost first; empty when unavailable. + fp_chain: Vec, + /// The copy filled its budget, so stack above it was not captured. + truncated: bool, + }, + + /// One executable file mapping observed in a tracked process (event pid = the process). + /// The runner resolves `path` to symbols + unwind data after the run. + ModuleMapping { + /// Runtime AVMA range and file offset, straight from /proc//maps. + start: u64, + end: u64, + offset: u64, + path: String, + }, + + /// Process name (/proc//comm) for the event's pid, emitted when first observed. + Comm { + name: String, + }, } +fn is_zero(value: &u64) -> bool { + *value == 0 +} pub struct MemtrackEventStream { deserializer: rmp_serde::Deserializer>, } @@ -120,7 +163,10 @@ mod tests { tid: 11, timestamp: 100, addr: 0x10, - kind: MemtrackEventKind::Malloc { size: 64 }, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, }, MemtrackEvent { pid: 1, @@ -167,21 +213,53 @@ mod tests { } let kinds = [ - MemtrackEventKind::Malloc { size: 7 }, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0, + }, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0xCAFE_BABE, + }, MemtrackEventKind::Free, MemtrackEventKind::Realloc { old_addr: Some(0x1000), size: 42, + stack_hash: 0, }, MemtrackEventKind::Realloc { old_addr: None, size: 42, + stack_hash: 0x1234, + }, + MemtrackEventKind::Calloc { + size: 9, + stack_hash: 0, + }, + MemtrackEventKind::AlignedAlloc { + size: 9, + stack_hash: 0, }, - MemtrackEventKind::Calloc { size: 9 }, - MemtrackEventKind::AlignedAlloc { size: 9 }, MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::StackDefinition { + hash: 0xDEAD_BEEF, + sp: 0x7FFF_0000, + regs: vec![0; 33], + bytes: vec![1, 2, 3, 4], + fp_chain: vec![0x1000, 0x2000], + truncated: false, + }, + MemtrackEventKind::ModuleMapping { + start: 0x400000, + end: 0x401000, + offset: 0, + path: "/usr/bin/app".to_string(), + }, + MemtrackEventKind::Comm { + name: "app".to_string(), + }, ]; for kind in kinds { @@ -190,7 +268,7 @@ mod tests { tid: 42, timestamp: 0xDEAD, addr: 0xBEEF, - kind, + kind: kind.clone(), }; let shadow = Shadow { pid: -7, @@ -207,6 +285,159 @@ mod tests { } } + #[test] + fn zero_stack_hash_serializes_byte_identically_to_legacy_shape() { + #[derive(serde::Serialize)] + #[serde(tag = "type")] + enum LegacyKind { + Malloc { + size: u64, + }, + Calloc { + size: u64, + }, + AlignedAlloc { + size: u64, + }, + Realloc { + #[serde(default, skip_serializing_if = "Option::is_none")] + old_addr: Option, + size: u64, + }, + } + + #[derive(serde::Serialize)] + struct LegacyEvent { + pid: libc::pid_t, + tid: libc::pid_t, + timestamp: u64, + addr: u64, + #[serde(flatten)] + kind: LegacyKind, + } + + let cases = [ + ( + MemtrackEventKind::Malloc { + size: 128, + stack_hash: 0, + }, + LegacyKind::Malloc { size: 128 }, + ), + ( + MemtrackEventKind::Calloc { + size: 256, + stack_hash: 0, + }, + LegacyKind::Calloc { size: 256 }, + ), + ( + MemtrackEventKind::AlignedAlloc { + size: 512, + stack_hash: 0, + }, + LegacyKind::AlignedAlloc { size: 512 }, + ), + ( + MemtrackEventKind::Realloc { + old_addr: Some(0x1000), + size: 1024, + stack_hash: 0, + }, + LegacyKind::Realloc { + old_addr: Some(0x1000), + size: 1024, + }, + ), + ( + MemtrackEventKind::Realloc { + old_addr: None, + size: 1024, + stack_hash: 0, + }, + LegacyKind::Realloc { + old_addr: None, + size: 1024, + }, + ), + ]; + + for (kind, legacy_kind) in cases { + let event = MemtrackEvent { + pid: 123, + tid: 456, + timestamp: 789, + addr: 0xABCD, + kind, + }; + let legacy = LegacyEvent { + pid: 123, + tid: 456, + timestamp: 789, + addr: 0xABCD, + kind: legacy_kind, + }; + + assert_eq!( + rmp_serde::to_vec(&event).unwrap(), + rmp_serde::to_vec(&legacy).unwrap() + ); + } + } + + #[test] + fn roundtrip_extended_events() -> anyhow::Result<()> { + let events = vec![ + MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 1000, + addr: 0, + kind: MemtrackEventKind::StackDefinition { + hash: 0xFEED_FACE_CAFE_BEEF, + sp: 0x7FFF_FFFF_0000, + regs: (0..33).map(|r| r * 0x100).collect(), + bytes: vec![0xDE, 0xAD, 0xBE, 0xEF, 0xAA, 0xBB, 0xCC, 0xDD], + fp_chain: vec![0x400500, 0x400600, 0x400700], + truncated: true, + }, + }, + MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 1001, + addr: 0, + kind: MemtrackEventKind::ModuleMapping { + start: 0x7FFF_0000, + end: 0x7FFF_1000, + offset: 0x2000, + path: "/lib/x86_64-linux-gnu/libc.so.6".to_string(), + }, + }, + MemtrackEvent { + pid: 42, + tid: 42, + timestamp: 1002, + addr: 0, + kind: MemtrackEventKind::Comm { + name: "worker-thread".to_string(), + }, + }, + ]; + + let artifact = MemtrackArtifact { + events: events.clone(), + }; + let mut buf = Vec::new(); + artifact.encode_to_writer(&mut buf)?; + + let stream = MemtrackArtifact::decode_streamed(Cursor::new(buf))?; + let collected: Vec<_> = stream.collect(); + assert_eq!(collected, events); + + Ok(()) + } + #[test] fn concatenated_frames_decode_in_order() -> anyhow::Result<()> { let events: Vec<_> = (0..2500) @@ -215,7 +446,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect(); @@ -265,7 +499,8 @@ mod tests { event.kind, MemtrackEventKind::Realloc { old_addr: None, - size: 42 + size: 42, + stack_hash: 0, } )); diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index c47b3aed..8cac46f0 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -94,7 +94,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect() } From b756e0ccf1d4e927bf77938e237f166787f9d965 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 16:13:41 +0200 Subject: [PATCH 07/16] fixup! feat(memtrack): enable stack capture through the tracker --- crates/memtrack/src/ebpf/attach_worker.rs | 17 +---- crates/memtrack/src/ebpf/memtrack/mod.rs | 57 +++++++++++---- crates/memtrack/src/ebpf/tracker.rs | 89 ++++++----------------- crates/memtrack/src/session.rs | 11 --- 4 files changed, 66 insertions(+), 108 deletions(-) diff --git a/crates/memtrack/src/ebpf/attach_worker.rs b/crates/memtrack/src/ebpf/attach_worker.rs index b2abf833..61a49c4e 100644 --- a/crates/memtrack/src/ebpf/attach_worker.rs +++ b/crates/memtrack/src/ebpf/attach_worker.rs @@ -12,7 +12,6 @@ use std::thread::JoinHandle; use std::time::Duration; use super::proc_fs::{Resolution, resolve_mapping, wait_all_stopped}; -use super::stacks::modules::ModuleTracker; const STOP_DEADLINE: Duration = Duration::from_secs(1); const POLL_INTERVAL_MS: u64 = 10; @@ -44,10 +43,7 @@ pub(crate) struct AttachWorker { } impl AttachWorker { - pub(crate) fn start( - bpf: Arc>, - modules: Option>>, - ) -> Result { + pub(crate) fn start(bpf: Arc>) -> Result { let shutdown = Arc::new(AtomicBool::new(false)); let fatal = Arc::new(Mutex::new(None)); let root_pid = Arc::new(AtomicI32::new(0)); @@ -62,7 +58,6 @@ impl AttachWorker { shutdown: shutdown.clone(), fatal: fatal.clone(), root_pid: root_pid.clone(), - modules, }; let handle = std::thread::spawn(move || worker.run()); @@ -131,7 +126,6 @@ struct Worker { shutdown: Arc, fatal: Arc>>, root_pid: Arc, - modules: Option>>, } impl Worker { @@ -217,15 +211,6 @@ impl Worker { } } - // Read mappings while the pids are stopped; short-lived processes may - // exit before their captured stacks reach userspace. - if let Some(modules) = &self.modules { - let mut modules = modules.lock(); - for pid in &stopped { - modules.refresh(*pid); - } - } - Ok(()) } diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 1ef5002d..13da1f72 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -248,20 +248,10 @@ impl MemtrackBpf { &self, poll_interval_ms: u64, tx: std::sync::mpsc::Sender, - stack_tx: Option>, ) -> Result { - let parse = move |data: &[u8]| { - if let Some(stack_tx) = &stack_tx - && let Some(record) = crate::ebpf::events::parse_alloc_stack_record(data) - { - let _ = stack_tx.send(record); - } - crate::ebpf::events::parse_event(data) - }; - with_skel!(self, skel => RingBufferPoller::new( &skel.maps.events, - parse, + crate::ebpf::events::parse_event, tx, poll_interval_ms, )) @@ -271,11 +261,52 @@ impl MemtrackBpf { pub(crate) fn poll_stack_definitions( &self, poll_interval_ms: u64, - tx: std::sync::mpsc::Sender, + tx: std::sync::mpsc::Sender, ) -> Result { + use libbpf_rs::MapCore; + let stack_traces = with_skel!(self, skel => { + libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) + .context("Failed to create handle for stack_traces map")? + }); + + let parse = move |data: &[u8]| -> Option { + let (mut event, stackid) = crate::ebpf::stacks::events::parse_stack_definition(data)?; + let fp_chain = if stackid < 0 { + Vec::new() + } else if let Ok(key) = u32::try_from(stackid) { + match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value + .chunks_exact(8) + .map(|word| { + u64::from_ne_bytes( + word.try_into().expect("chunks_exact yields 8 bytes"), + ) + }) + .take_while(|&address| address != 0) + .collect(), + Ok(None) => Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + Vec::new() + } + } + } else { + Vec::new() + }; + + if let runner_shared::artifacts::MemtrackEventKind::StackDefinition { + fp_chain: ref mut chain, + .. + } = event.kind + { + *chain = fp_chain; + } + Some(event) + }; + with_skel!(self, skel => RingBufferPoller::new( &skel.maps.stack_defs, - crate::ebpf::stacks::events::parse_stack_definition, + parse, tx, poll_interval_ms, )) diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 7dc48773..db067799 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,9 +1,7 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; use crate::ebpf::stacks::config::StackCaptureConfig; -use crate::ebpf::stacks::modules::ModuleTracker; -use crate::ebpf::stacks::recorder::StackRecorder; -use crate::ebpf::stacks::report::StackReport; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; @@ -11,7 +9,6 @@ use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc; pub struct Tracker { @@ -19,10 +16,7 @@ pub struct Tracker { worker: Mutex>, allocators: bool, stack_config: Mutex>, - stack_modules: Option>>, - stack_recorder: Mutex>, - stack_report: Mutex>, - live_sessions: Arc, + stack_capture_enabled: bool, } impl Tracker { @@ -78,24 +72,19 @@ impl Tracker { } let bpf = Arc::new(Mutex::new(bpf)); - let stack_modules = stacks - .as_ref() - .map(|_| Arc::new(Mutex::new(ModuleTracker::new()))); let worker = if allocators { - Some(AttachWorker::start(bpf.clone(), stack_modules.clone())?) + Some(AttachWorker::start(bpf.clone())?) } else { None }; + let stack_capture_enabled = stacks.is_some(); Ok(Self { bpf, worker: Mutex::new(worker), allocators, stack_config: Mutex::new(stacks), - stack_modules, - stack_recorder: Mutex::new(None), - stack_report: Mutex::new(None), - live_sessions: Arc::new(AtomicUsize::new(0)), + stack_capture_enabled, }) } @@ -108,9 +97,11 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { - if self.stack_modules.is_some() && self.stack_config.lock().is_none() { + if self.stack_capture_enabled && self.stack_config.lock().is_none() { bail!("stack capture supports a single spawned command per tracker"); } + let stack_capture_on = self.stack_config.lock().take().is_some(); + let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); @@ -118,6 +109,8 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; + + let (tx, rx) = mpsc::channel(); match self.worker.lock().as_ref() { Some(worker) => worker.set_root_pid(pid), // No watcher to arm means exec mappings would be missed. @@ -125,40 +118,19 @@ impl Tracker { None => {} } - let stack_records = if let Some(config) = self.stack_config.lock().take() { - let Some(modules) = self.stack_modules.clone() else { - bail!("stack capture module tracker is unavailable"); - }; - let (recorder, tx) = StackRecorder::start(config, modules); - *self.stack_recorder.lock() = Some(recorder); - Some(tx) - } else { - None - }; - - let (tx, rx) = mpsc::channel(); let (poller, stack_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; - let stack_poller = match &stack_records { - Some(tx) => Some(bpf.poll_stack_definitions(10, tx.clone())?), - None => None, + let stack_poller = if stack_capture_on { + Some(bpf.poll_stack_definitions(10, tx.clone())?) + } else { + None }; - ( - bpf.poll_events_with_channel(10, tx, stack_records)?, - stack_poller, - ) + (bpf.poll_events_with_channel(10, tx)?, stack_poller) }; resume(pid)?; - self.live_sessions.fetch_add(1, Ordering::SeqCst); - Ok(Session::new( - child, - rx, - poller, - stack_poller, - self.live_sessions.clone(), - )) + Ok(Session::new(child, rx, poller, stack_poller)) } /// Enable allocator-event tracking in the BPF program. Lifetime events @@ -179,6 +151,11 @@ impl Tracker { self.bpf.lock().dropped_events_count() } + /// Read the stack capture counters from BPF map. + pub fn stack_capture_stats(&self) -> Result { + self.bpf.lock().stack_capture_stats() + } + /// Only meaningful while the BPF object is alive; teardown frees the maps. pub fn ownership_maps(&self) -> Result { self.bpf.lock().ownership_maps() @@ -186,37 +163,13 @@ impl Tracker { /// Stop the attach worker, if any, and surface any fatal error it recorded, /// including missed exec mappings (incomplete allocator coverage). - /// - /// All [`Session`] values must be dropped before calling this method. It - /// returns an error otherwise because the recorder cannot finish while its - /// ring pollers hold channel senders. pub fn finish(&self) -> Result<()> { - let live_sessions = self.live_sessions.load(Ordering::SeqCst); - if live_sessions != 0 { - bail!( - "{live_sessions} session(s) still alive; drop them first because the recorder cannot finish while ring pollers hold channel senders" - ); - } - if let Some(recorder) = self.stack_recorder.lock().take() { - let bpf = self.bpf.lock(); - let stats = bpf.stack_capture_stats()?; - let report = recorder.finish(stats, |stackid| bpf.fp_chain(stackid))?; - drop(bpf); - *self.stack_report.lock() = Some(report); - } - match self.worker.lock().take() { Some(worker) => worker.finish(), None => Ok(()), } } - /// The stack-capture report produced by [`Self::finish`]. Can only be taken - /// once, and is `None` when capture was off. - pub fn take_stack_report(&self) -> Option { - self.stack_report.lock().take() - } - /// Detach all attached probes. Called explicitly at teardown because the /// process may exit without ever dropping the tracker (the IPC thread holds /// an Arc clone), in which case the kernel would close each link fd serially. diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 91522c36..9bed66b7 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -2,8 +2,6 @@ use crate::ebpf::poller::RingBufferPoller; use crate::prelude::*; use runner_shared::artifacts::MemtrackEvent; use std::process::{Child, ExitStatus}; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::mpsc::Receiver; /// A spawned, tracked process together with its event pipeline. The pipeline @@ -13,7 +11,6 @@ pub struct Session { events: Option>, _poller: RingBufferPoller, _stack_poller: Option, - live_sessions: Arc, } impl Session { @@ -22,14 +19,12 @@ impl Session { events: Receiver, poller: RingBufferPoller, stack_poller: Option, - live_sessions: Arc, ) -> Self { Self { child, events: Some(events), _poller: poller, _stack_poller: stack_poller, - live_sessions, } } @@ -47,9 +42,3 @@ impl Session { Ok(self.child.wait()?) } } - -impl Drop for Session { - fn drop(&mut self) { - self.live_sessions.fetch_sub(1, Ordering::SeqCst); - } -} From dcd1d905a35b9886247b149dc0eaa5b3fd1a8608 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 16:14:18 +0200 Subject: [PATCH 08/16] fixup! test(memtrack): cover allocation stack capture --- crates/memtrack/tests/dlopen_tests.rs | 8 +- crates/memtrack/tests/shared.rs | 29 +-- crates/memtrack/tests/stack_tests.rs | 353 ++++++++++++++------------ 3 files changed, 204 insertions(+), 186 deletions(-) diff --git a/crates/memtrack/tests/dlopen_tests.rs b/crates/memtrack/tests/dlopen_tests.rs index cbabfeaf..ec243ae2 100644 --- a/crates/memtrack/tests/dlopen_tests.rs +++ b/crates/memtrack/tests/dlopen_tests.rs @@ -67,13 +67,13 @@ fn test_dlopen_allocator() -> Result<(), Box> { let malloc_addrs: HashSet = events .iter() .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size: 4242 } => Some(e.addr), + MemtrackEventKind::Malloc { size: 4242, .. } => Some(e.addr), _ => None, }) .collect(); let malloc_count = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let free_count = events .iter() @@ -125,11 +125,11 @@ fn test_thread_dlopen() -> Result<(), Box> { |events| { let m4242 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let m4243 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243, .. })) .count(); assert_eq!(m4242, 100, "expected 100 mi_malloc(4242) events"); diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 16d9fad1..903947fb 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -25,7 +25,7 @@ macro_rules! assert_events_snapshot { let formatted_events: Vec = $events .iter() .filter(|e| { - // RSS and lifecycle events are asserted by dedicated tests, not snapshots. + // RSS, lifecycle, stack, and mapping events are asserted by dedicated tests, not snapshots. !matches!( e.kind, MemtrackEventKind::Rss { .. } @@ -33,6 +33,9 @@ macro_rules! assert_events_snapshot { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + | MemtrackEventKind::StackDefinition { .. } + | MemtrackEventKind::ModuleMapping { .. } + | MemtrackEventKind::Comm { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -93,6 +96,9 @@ macro_rules! assert_events_with_marker_for_each_variant { /// old address. pub fn describe_kind(kind: &MemtrackEventKind) -> String { match kind { + MemtrackEventKind::Malloc { size, .. } => format!("Malloc {{ size: {size} }}"), + MemtrackEventKind::Calloc { size, .. } => format!("Calloc {{ size: {size} }}"), + MemtrackEventKind::AlignedAlloc { size, .. } => format!("AlignedAlloc {{ size: {size} }}"), MemtrackEventKind::Realloc { size, .. } => format!("Realloc {{ size: {size} }}"), other => format!("{other:?}"), } @@ -106,7 +112,7 @@ pub fn between_markers(events: &[Event]) -> Vec { const MARKER: u64 = 0xC0D5_9EED; let is_marker = - |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size } if size == MARKER); + |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size, .. } if size == MARKER); events .iter() @@ -122,6 +128,9 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + | MemtrackEventKind::StackDefinition { .. } + | MemtrackEventKind::ModuleMapping { .. } + | MemtrackEventKind::Comm { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -221,22 +230,12 @@ pub fn track_command_with_rmap_maps( let maps = tracker.ownership_maps()?; Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } -/// Track a command with allocation stack capture enabled, returning its events -/// and stack report. +/// Track a command with allocation stack capture enabled, returning its events. pub fn track_command_with_stacks( command: Command, config: memtrack::StackCaptureConfig, -) -> anyhow::Result<( - Vec, - memtrack::StackReport, - std::thread::JoinHandle<()>, -)> { - let tracker = Tracker::new_with_stack_capture(config)?; - let (tracker, events, ()) = run_tracked(command, tracker, |_, _| Ok(()))?; - let report = tracker - .take_stack_report() - .context("tracker did not produce a stack report")?; - Ok((events, report, std::thread::spawn(move || drop(tracker)))) +) -> TrackResult { + track_command_with_tracker(command, Tracker::new_with_stack_capture(config)?) } /// Track a command with rmap hooks and snapshot the ownership maps at a diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 7e1fe24d..c7a13db0 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -1,10 +1,11 @@ #[macro_use] mod shared; +use runner_shared::artifacts::MemtrackEventKind; use std::collections::HashSet; -use std::path::Path; use std::process::Command; use tempfile::TempDir; + #[test_with::env(GITHUB_ACTIONS)] #[test_log::test] fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { @@ -14,153 +15,90 @@ fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::StackDefinition { + hash, + sp, + regs, + bytes, + fp_chain, + truncated, + } => Some((e, *hash, *sp, regs, bytes, fp_chain, *truncated)), + _ => None, + }) + .collect(); + assert!( - report.definitions.len() >= 2, + definitions.len() >= 2, "expected at least two stack definitions, got {} ({} events)", - report.definitions.len(), + definitions.len(), events.len() ); - let hashes: HashSet<_> = report - .definitions - .iter() - .map(|definition| definition.hash) - .collect(); + + let hashes: HashSet = definitions.iter().map(|(_, hash, ..)| *hash).collect(); assert_eq!( hashes.len(), - report.definitions.len(), + definitions.len(), "stack definitions must be deduplicated by unique hash" ); + assert!( - report - .definitions + definitions .iter() - .all(|definition| definition.copy_len > 0), + .all(|(_, _, _, _, bytes, _, _)| !bytes.is_empty()), "every stack definition must contain copied stack bytes" ); assert!( - report - .definitions - .iter() - .all(|definition| definition.sp != 0), + definitions.iter().all(|(_, _, sp, ..)| *sp != 0), "every stack definition must include a nonzero stack pointer" ); - // `truncated` means the budget ran out, not that the copy was short: a copy - // that stops at the top of the stack mapping is complete however short it is. - // The kernel only ever copies whole chunks, so the emitted length is exactly - // the hashed length. - // - // Whether any capture is complete at this budget depends on how much stack - // is readable above the stack pointer, which shifts with the environment - // size, so only the maximum budget can guarantee completeness. assert!( - report.definitions.iter().all(|definition| { - definition.copy_len <= copy_size as usize - && definition.copy_len % 512 == 0 - && definition.truncated == (definition.copy_len == copy_size as usize) - }), - "stack copies must be whole chunks within the budget and flagged truncated only when they fill it: {:?}", - report - .definitions + definitions .iter() - .map(|definition| (definition.copy_len, definition.truncated)) - .collect::>() - ); - assert_eq!( - report.stats.copy_failed, 0, - "stack capture reported {} copy failures", - report.stats.copy_failed - ); - assert_eq!( - report.stats.ring_full, 0, - "stack definition ring buffer reported {} full events", - report.stats.ring_full + .all(|(_, _, _, regs, ..)| regs.len() == 33), + "every stack definition must include 33 registers on x86_64" ); assert!( - report.alloc_events_with_stack > 0, - "expected allocation events carrying captured stack hashes" + definitions.iter().all(|(_, _, _, _, bytes, _, truncated)| { + bytes.len() <= copy_size as usize + && bytes.len() % 512 == 0 + && *truncated == (bytes.len() == copy_size as usize) + }), + "stack copies must be whole 512-byte chunks within copy_size and flagged truncated only when full" ); - // Every chain starts inside the allocator the probe sits on, so the paths - // only diverge from the second frame onwards. - let all_stackids_negative = report - .definitions + let alloc_hashes: Vec = events .iter() - .all(|definition| definition.stackid < 0); - if all_stackids_negative { - assert!( - report - .definitions - .iter() - .all(|definition| definition.fp_chain.is_empty()), - "all stack IDs are negative, so every FP chain must be empty" - ); - } else { - let chains: HashSet<_> = report - .definitions - .iter() - .map(|definition| definition.fp_chain.clone()) - .collect(); - assert!( - chains.len() >= 2, - "some stack IDs are available, but the {} definitions produced only {} distinct frame-pointer chains: {:?}", - report.definitions.len(), - chains.len(), - chains - ); - } - - let fixture_pid = report - .definitions - .first() - .expect("at least one definition was asserted above") - .pid; - let modules = report - .modules - .get(&fixture_pid) - .expect("stack report must include module mappings for the fixture pid"); - let fixture_name = binary - .file_name() - .expect("compiled fixture must have a file name") - .to_string_lossy() - .into_owned(); + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } => { + if stack_hash != 0 { + Some(stack_hash) + } else { + None + } + } + _ => None, + }) + .collect(); + assert!( - modules.iter().any(|mapping| { - Path::new(&mapping.path) - .file_name() - .is_some_and(|name| name.to_string_lossy() == fixture_name.as_str()) - }), - "module mappings for pid {fixture_pid} must include fixture binary {fixture_name:?}" + !alloc_hashes.is_empty(), + "expected at least one allocation event carrying a captured stack hash" ); assert!( - modules.iter().any(|mapping| { - Path::new(&mapping.path) - .file_name() - .is_some_and(|name| name.to_string_lossy().starts_with("libc")) - }), - "module mappings for pid {fixture_pid} must include a libc image" - ); - - let dumped: serde_json::Value = serde_json::from_reader(std::fs::File::open(&dump_path)?) - .map_err(|error| { - format!( - "stack dump at {} did not parse: {error}", - dump_path.display() - ) - })?; - assert_eq!( - dumped["unique_stacks"].as_u64(), - Some(report.unique_stacks as u64), - "the written dump must agree with the in-memory report" + alloc_hashes.iter().all(|hash| hashes.contains(hash)), + "every non-zero stack_hash on an alloc must have a matching StackDefinition hash" ); thread_handle @@ -178,18 +116,36 @@ fn dedup_collapses_repeated_call_paths() -> Result<(), Box = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::StackDefinition { hash, .. } => Some(*hash), + _ => None, + }) + .collect(); + + let alloc_with_stack_count = events + .iter() + .filter(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } => stack_hash != 0, + _ => false, + }) + .count(); + assert!( - report.alloc_events_with_stack > report.unique_stacks as u64, - "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", - report.alloc_events_with_stack, - report.unique_stacks, + alloc_with_stack_count > def_hashes.len(), + "expected repeated call paths to deduplicate raw stacks: {alloc_with_stack_count} stack-bearing events across {} unique stacks ({} total events)", + def_hashes.len(), events.len() ); @@ -214,36 +170,31 @@ fn max_copy_budget_loads_and_captures_whole_stacks() -> Result<(), Box = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::StackDefinition { + hash, + bytes, + truncated, + .. + } => Some((*hash, bytes.len(), *truncated)), + _ => None, + }) + .collect(); + assert!( - report.copy_size > memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE, - "the clamp must land above the default budget, got {}", - report.copy_size - ); - assert!( - !report.definitions.is_empty(), + !definitions.is_empty(), "expected stack definitions at the maximum copy budget" ); assert!( - report - .definitions - .iter() - .all(|definition| !definition.truncated), - "no capture can be budget-limited at the maximum budget: {:?}", - report - .definitions - .iter() - .map(|definition| (definition.copy_len, definition.truncated)) - .collect::>() - ); - assert_eq!( - report.stats.copy_failed, 0, - "stack capture reported {} copy failures at the maximum budget", - report.stats.copy_failed + definitions.iter().all(|(_, _, truncated)| !*truncated), + "no capture can be budget-limited at the maximum budget: {definitions:?}" ); thread_handle @@ -264,18 +215,17 @@ fn exhausted_stack_trace_map_costs_only_fp_chains() -> Result<(), Box Result<(), Box stack_hash != 0, + _ => false, + }) + .count(); + + let allocs_with_stack = events + .iter() + .filter(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } => stack_hash != 0, + _ => false, + }) + .count(); - assert!( - report.stats.stackid_failed > 0, - "a one-slot frame-pointer map must report failed walks, got {}", - report.stats.stackid_failed - ); - assert_eq!( - report.alloc_events_with_stack, baseline_report.alloc_events_with_stack, - "allocation totals must be unaffected by frame-pointer map exhaustion" - ); assert_eq!( - report.alloc_events_without_stack, 0, - "no allocation may lose its stack identity when only the walk fails" + allocs_with_stack, baseline_allocs_with_stack, + "allocation totals with stack must be unaffected by frame-pointer map exhaustion" ); + + let baseline_def_hashes: HashSet = baseline_events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::StackDefinition { hash, .. } => Some(*hash), + _ => None, + }) + .collect(); + + let def_hashes: HashSet = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::StackDefinition { hash, .. } => Some(*hash), + _ => None, + }) + .collect(); + assert_eq!( - report.unique_stacks, baseline_report.unique_stacks, + def_hashes.len(), + baseline_def_hashes.len(), "stack identity comes from the copied bytes, not the frame-pointer walk" ); assert_eq!( - report.stats.copy_failed, 0, - "stack copies must still succeed with an exhausted frame-pointer map" + def_hashes, baseline_def_hashes, + "distinct stack definition hashes must match baseline" ); - thread_handle - .join() - .expect("tracker teardown thread panicked"); Ok(()) } @@ -324,11 +305,49 @@ fn default_off_leaves_allocation_events_intact() -> Result<(), Box = events + .iter() + .filter(|e| { + matches!( + e.kind, + MemtrackEventKind::Malloc { .. } + | MemtrackEventKind::Calloc { .. } + | MemtrackEventKind::AlignedAlloc { .. } + | MemtrackEventKind::Realloc { .. } + | MemtrackEventKind::Free + ) + }) + .collect(); + assert!( - !events.is_empty(), + !alloc_events.is_empty(), "default-off tracking must still report allocation events" ); + let stack_defs_count = events + .iter() + .filter(|e| matches!(e.kind, MemtrackEventKind::StackDefinition { .. })) + .count(); + assert_eq!( + stack_defs_count, 0, + "default-off tracking must emit zero StackDefinition events" + ); + + let non_zero_stack_hashes = events + .iter() + .filter(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } => stack_hash != 0, + _ => false, + }) + .count(); + assert_eq!( + non_zero_stack_hashes, 0, + "default-off tracking must have zero non-zero stack_hash on allocation events" + ); + thread_handle .join() .expect("tracker teardown thread panicked"); From b3269b47f51cb5fdd5a1ea7bd8b64253bb9f3f64 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 17:08:13 +0200 Subject: [PATCH 09/16] fixup! feat(memtrack): add userspace stack-capture module --- crates/memtrack/src/ebpf/stacks/config.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs index f7ed299d..59ea677a 100644 --- a/crates/memtrack/src/ebpf/stacks/config.rs +++ b/crates/memtrack/src/ebpf/stacks/config.rs @@ -11,12 +11,13 @@ impl StackCaptureConfig { pub const DEFAULT_COPY_SIZE: u32 = 8192; pub const DEFAULT_STACK_TRACE_CAPACITY: u32 = 16384; - /// Returns `None` unless stack capture was explicitly enabled. + /// Returns `None` only when stack capture was explicitly disabled with + /// `CODSPEED_MEMTRACK_CAPTURE_STACKS=0`; capture is on by default. pub fn from_env() -> Option { if std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS") .ok() .as_deref() - != Some("1") + == Some("0") { return None; } From 888dbf8cc3d766d101bcb2fc326c8a8df53819b7 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 17:08:14 +0200 Subject: [PATCH 10/16] fixup! test(memtrack): cover allocation stack capture --- crates/memtrack/tests/stack_tests.rs | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index c7a13db0..c99e84c5 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -294,15 +294,35 @@ fn exhausted_stack_trace_map_costs_only_fp_chains() -> Result<(), Box Self { + // SAFETY: tests run with --test-threads 1, so no concurrent env access. + unsafe { std::env::set_var("CODSPEED_MEMTRACK_CAPTURE_STACKS", "0") }; + Self + } +} + +impl Drop for DisableCaptureGuard { + fn drop(&mut self) { + // SAFETY: see `set`. + unsafe { std::env::remove_var("CODSPEED_MEMTRACK_CAPTURE_STACKS") }; + } +} + #[test_with::env(GITHUB_ACTIONS)] #[test_log::test] -fn default_off_leaves_allocation_events_intact() -> Result<(), Box> { +fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { let temp_dir = TempDir::new()?; let binary = shared::compile_c_source( include_str!("../testdata/stack_paths.c"), "stack_paths_disabled", temp_dir.path(), )?; + let _guard = DisableCaptureGuard::set(); let (events, thread_handle) = shared::track_binary(&binary)?; let alloc_events: Vec<_> = events @@ -321,7 +341,7 @@ fn default_off_leaves_allocation_events_intact() -> Result<(), Box Result<(), Box Result<(), Box Date: Fri, 28 Aug 2026 16:36:01 +0200 Subject: [PATCH 11/16] fixup! feat(memtrack): capture allocation stacks in eBPF --- crates/memtrack/src/ebpf/c/stack_capture.bpf.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h index 090fd6b1..019cbb80 100644 --- a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -67,11 +67,13 @@ struct { __type(value, struct stack_def_scratch); } stack_scratch SEC(".maps"); +/* The value is 64-bit because the BPF backend of older clang cannot select a + * 32-bit atomic compare-and-swap. */ struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(max_entries, 1); __type(key, __u32); - __type(value, __u32); + __type(value, __u64); } stack_busy SEC(".maps"); static __always_inline void bump_stack_counter(__u32 index) { @@ -231,7 +233,7 @@ static __always_inline void capture_stack(struct pt_regs* ctx) { } __u32 zero = 0; - __u32* busy = bpf_map_lookup_elem(&stack_busy, &zero); + __u64* busy = bpf_map_lookup_elem(&stack_busy, &zero); if (!busy) { return; } From ea0fe74428eaade909ec2d5f519cf80202e1dbe7 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 16:36:12 +0200 Subject: [PATCH 12/16] fixup! test(memtrack): cover allocation stack capture --- crates/memtrack/tests/stack_tests.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index c99e84c5..6cb63a5f 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -281,15 +281,13 @@ fn exhausted_stack_trace_map_costs_only_fp_chains() -> Result<(), Box Date: Fri, 28 Aug 2026 17:05:51 +0200 Subject: [PATCH 13/16] fixup! feat(memtrack): capture allocation stacks in eBPF --- crates/memtrack/src/ebpf/c/allocator.h | 49 ++++---- crates/memtrack/src/ebpf/c/event.h | 11 +- .../memtrack/src/ebpf/c/stack_capture.bpf.h | 106 +++++++++--------- .../memtrack/src/ebpf/c/utils/event_helpers.h | 7 +- 4 files changed, 90 insertions(+), 83 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 25a3014c..8de96317 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -5,28 +5,26 @@ #include "utils/map_helpers.h" #include "utils/process_tracking.h" -#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ - BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ - SEC(UPROBE_SEC) \ - int uprobe_##name(struct pt_regs* ctx) { \ - capture_stack(ctx); \ - return store_param(&name##_arg, arg_expr); \ - } \ - SEC(URETPROBE_SEC) \ - int uretprobe_##name(struct pt_regs* ctx) { \ - __u64* arg_ptr = take_param(&name##_arg); \ - if (!arg_ptr) { \ - return 0; \ - } \ - /* The slot is per-thread but shared by allocators, so every return must \ - * clear it before failure can strand a nested call's identity. */ \ - __u64 stack_hash = take_stack_hash(); \ - __u64 ret_val = PT_REGS_RC(ctx); \ - if (ret_val == 0) { \ - return 0; \ - } \ - __u64 arg0 = *arg_ptr; \ - submit_block; \ +#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ + BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ + SEC(UPROBE_SEC) \ + int uprobe_##name(struct pt_regs* ctx) { \ + stash_stack_hash(capture_stack(ctx)); \ + return store_param(&name##_arg, arg_expr); \ + } \ + SEC(URETPROBE_SEC) \ + int uretprobe_##name(struct pt_regs* ctx) { \ + __u64* arg_ptr = take_param(&name##_arg); \ + if (!arg_ptr) { \ + return 0; \ + } \ + __u64 stack_hash = take_stack_hash(); \ + __u64 ret_val = PT_REGS_RC(ctx); \ + if (ret_val == 0) { \ + return 0; \ + } \ + __u64 arg0 = *arg_ptr; \ + submit_block; \ } #define UPROBE_RET(name, arg_expr, submit_block) \ @@ -36,6 +34,7 @@ if (arg0 == 0) { \ return 0; \ } \ + __u64 stack_hash = capture_stack(ctx); \ submit_block; \ } @@ -54,7 +53,7 @@ return 0; \ } \ \ - capture_stack(ctx); \ + stash_stack_hash(capture_stack(ctx)); \ \ struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr}; \ \ @@ -87,7 +86,7 @@ UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); }) +UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0, stack_hash); }) UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx), { return submit_calloc_event(arg0, ret_val, stack_hash); }) @@ -124,7 +123,7 @@ int uprobe_posix_memalign(struct pt_regs* ctx) { return 0; } - capture_stack(ctx); + stash_stack_hash(capture_stack(ctx)); struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)}; bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY); diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index bc0bc43b..f4b33c75 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -42,9 +42,9 @@ struct stack_regs { uint64_t reg[MEMTRACK_STACK_REGS]; }; -/* Head of a stack-definition record; `copy_len` raw stack bytes read upwards - * from `sp` follow it. */ -struct stack_def_header { +/* Head of a stack record; `copy_len` raw stack bytes read upwards from `sp` + * follow it. */ +struct stack_header { uint64_t hash; uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ @@ -52,7 +52,7 @@ struct stack_def_header { uint32_t pid; uint32_t tid; uint32_t copy_len; - uint8_t truncated; /* the copy hit the size cap or a short read */ + uint8_t truncated; /* the copy hit the size cap */ uint8_t _pad[3]; struct stack_regs regs; }; @@ -78,7 +78,8 @@ struct event { /* Deallocation event (free) */ struct { - uint64_t addr; /* address to free */ + uint64_t addr; /* address to free */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } free; /* Reallocation event - includes both old and new addresses */ diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h index 019cbb80..28f7fcf8 100644 --- a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -5,31 +5,23 @@ #include "utils/map_helpers.h" #include "utils/process_tracking.h" -/* == Allocation stack capture == +/* At allocator entry the caller's raw user stack is copied and hashed; the hash + * travels on the allocation event as its stack identity. The first time a hash + * is seen, the copied bytes plus a register snapshot are emitted as a stack + * record so the stack can be DWARF-unwound offline, with an in-kernel + * frame-pointer walk alongside as the fallback. * - * At allocator entry the caller's raw user stack is copied and hashed; the hash - * travels on the allocation event and identifies the call path. The first time - * a hash is seen, the copied bytes plus a full user register snapshot are - * emitted as a stack-definition record so the stack can be DWARF-unwound - * offline. An in-kernel frame-pointer walk rides along in the definition as the - * fallback for binaries whose .eh_frame is missing or whose DWARF unwind - * truncates. - * - * A 64-bit FNV-1a digest means distinct byte sequences alias with birthday - * probability roughly n^2/2^65: negligible at the thousands of identities a - * run produces, but not zero. It also splits one call path into several - * identities whenever the locals and arguments living in the copied region - * differ. That is a deliberate trade: aliasing corrupts attribution, splitting - * only costs definition records. + * Hashing raw bytes rather than unwound frames splits one call path into + * several identities whenever locals or arguments in the copied region differ. + * That only costs extra records; the reverse trade (aliasing) would corrupt + * attribution. */ const volatile __u8 capture_stacks_enabled = 0; const volatile __u32 stack_copy_size = 8192; #define STACK_TRACE_MAX_DEPTH 127 -/* Chunk granularity of the stack copy: the recovered length is exact to within - * one chunk, and a full-budget copy costs MEMTRACK_MAX_STACK_COPY/chunk helper - * calls. MEMTRACK_MAX_STACK_COPY is a whole multiple of it. */ +/* Copy granularity: a recovered length is exact only to within one chunk. */ #define STACK_COPY_CHUNK 512 #define FNV64_OFFSET 0xcbf29ce484222325ULL #define FNV64_PRIME 0x00000100000001b3ULL @@ -42,18 +34,18 @@ struct { __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); } stack_traces SEC(".maps"); -/* Definitions are bulky but rare (one per distinct hash), so they get their own +/* Records are bulky but rare (one per distinct hash), so they get their own * ring rather than inflating the fixed-size `struct event` path. */ -BPF_RINGBUF(stack_defs, 64 * 1024 * 1024); +BPF_RINGBUF(stacks, 64 * 1024 * 1024); BPF_HASH_MAP(seen_stack_hashes, __u64, __u8, 65536); BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); -/* The definition record is built in place here and handed to the ring buffer as - * one contiguous variable-length blob. `words` aliases `bytes` so the hash loop +/* The record is built in place here and handed to the ring buffer as one + * contiguous variable-length blob. `words` aliases `bytes` so the hash loop * reads whole registers without a per-byte shift chain. */ -struct stack_def_scratch { - struct stack_def_header header; +struct stack_scratch_buf { + struct stack_header header; union { __u8 bytes[MEMTRACK_MAX_STACK_COPY]; __u64 words[MEMTRACK_MAX_STACK_COPY / 8]; @@ -64,7 +56,7 @@ struct { __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); __uint(max_entries, 1); __type(key, __u32); - __type(value, struct stack_def_scratch); + __type(value, struct stack_scratch_buf); } stack_scratch SEC(".maps"); /* The value is 64-bit because the BPF backend of older clang cannot select a @@ -117,13 +109,12 @@ static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_re #error "stack capture needs a DWARF register mapping for this architecture" #endif -/* Copy and hash the caller's stack, stashing the identity for the matching - * uretprobe. Emit a definition on the first occurrence. */ -static __always_inline void capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { +/* Returns the stack identity, or 0 when nothing could be copied. */ +static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { __u32 zero = 0; - struct stack_def_scratch* scratch = bpf_map_lookup_elem(&stack_scratch, &zero); + struct stack_scratch_buf* scratch = bpf_map_lookup_elem(&stack_scratch, &zero); if (!scratch) { - return; + return 0; } __u64 sp = PT_REGS_SP(ctx); @@ -138,14 +129,12 @@ static __always_inline void capture_stack_inner(struct pt_regs* ctx, struct task /* bpf_probe_read_user() is all-or-nothing and the readable region ends at * the top of the stack mapping, which is not knowable up front, so the copy - * advances in chunks and stops at the first unreadable one. A failed chunk - * can leave up to STACK_COPY_CHUNK - 1 readable bytes at the top of the - * mapping uncopied, so a complete copy is exact only to chunk granularity. + * advances in chunks and stops at the first unreadable one. * - * Each chunk is hashed as it lands, with a constant iteration count the - * compiler fully unrolls. A single loop over the whole copy instead costs - * the verifier a state fork per word and blows the one-million instruction - * budget well before the maximum copy size. */ + * Each chunk is hashed as it lands, over a constant iteration count the + * compiler fully unrolls. One loop over the whole copy instead costs the + * verifier a state fork per word and blows the one-million instruction + * budget well below the maximum copy size. */ __u64 hash = FNV64_OFFSET; __u32 got = 0; #pragma clang loop unroll(disable) @@ -168,7 +157,7 @@ static __always_inline void capture_stack_inner(struct pt_regs* ctx, struct task if (got == 0) { bump_stack_counter(MEMTRACK_STACK_COUNTER_COPY_FAILED); - return; + return 0; } __u8 truncated = got >= want; @@ -184,13 +173,10 @@ static __always_inline void capture_stack_inner(struct pt_regs* ctx, struct task hash = FNV64_OFFSET; } - __u64 tid = ids.tid; - bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); - __u8 marker = 1; long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); - if (gate_result == -17) { - return; + if (gate_result == -17) { /* -EEXIST: already emitted */ + return hash; } if (gate_result != 0) { /* A full gate cannot retain this identity, so emit it on every @@ -216,42 +202,60 @@ static __always_inline void capture_stack_inner(struct pt_regs* ctx, struct task scratch->header._pad[2] = 0; fill_stack_regs(&scratch->header.regs, ctx); - if (bpf_ringbuf_output(&stack_defs, scratch, sizeof(struct stack_def_header) + got, 0) != 0) { + if (bpf_ringbuf_output(&stacks, scratch, sizeof(struct stack_header) + got, 0) != 0) { bump_stack_counter(MEMTRACK_STACK_COUNTER_RING_FULL); bpf_map_delete_elem(&seen_stack_hashes, &hash); } + + return hash; } -static __always_inline void capture_stack(struct pt_regs* ctx) { +/* Copy and hash the caller's stack, emitting a record on first sight of the + * resulting identity. Returns 0 when capture is off or nothing was copied. */ +static __always_inline __u64 capture_stack(struct pt_regs* ctx) { if (!capture_stacks_enabled || !is_enabled()) { - return; + return 0; } struct task_ids ids = current_task_ids(); if (!is_tracked(ids.tgid)) { - return; + return 0; } __u32 zero = 0; __u64* busy = bpf_map_lookup_elem(&stack_busy, &zero); if (!busy) { - return; + return 0; } + /* uprobe_multi runs programs without the bpf_prog_active recursion guard, + * so a task preempting this one on the same CPU could corrupt the scratch. */ if (__sync_val_compare_and_swap(busy, 0, 1) != 0) { bump_stack_counter(MEMTRACK_STACK_COUNTER_PREEMPTED); - return; + return 0; } - capture_stack_inner(ctx, ids); + __u64 hash = capture_stack_inner(ctx, ids); /* A plain store, not an atomic release: the only contender is a task that * preempted this one on this same CPU, and the context switch between them * already orders the write. The BPF backend cannot select a release store. */ *busy = 0; + return hash; +} + +/* Hand an identity to the matching uretprobe. */ +static __always_inline void stash_stack_hash(__u64 hash) { + if (hash == 0) { + return; + } + + __u64 tid = current_tid(); + bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); } /* The identity stashed by the matching entry probe, or 0 when capture is off or - * the entry probe bailed out. */ + * the entry probe bailed out. The slot is per-thread but shared by allocators, + * so every return path must clear it. */ static __always_inline __u64 take_stack_hash(void) { if (!capture_stacks_enabled) { return 0; diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index 2cbd83d7..ca53593d 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -113,8 +113,11 @@ static __always_inline int submit_calloc_event(__u64 size, __u64 addr, __u64 sta }); } -static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); +static __always_inline int submit_free_event(__u64 addr, __u64 stack_hash) { + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { + e->data.free.addr = addr; + e->data.free.stack_hash = stack_hash; + }); } static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size, From 877913afc617136f3de2a79b8a1f433115a81967 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 17:06:02 +0200 Subject: [PATCH 14/16] fixup! feat(memtrack): add userspace stack-capture module --- crates/memtrack/src/ebpf/events.rs | 7 +- crates/memtrack/src/ebpf/mod.rs | 2 +- crates/memtrack/src/ebpf/stacks/config.rs | 72 ++----- crates/memtrack/src/ebpf/stacks/counters.rs | 48 ++--- crates/memtrack/src/ebpf/stacks/events.rs | 126 +++++------ .../runner-shared/benches/memtrack_writer.rs | 37 +++- .../src/artifacts/memtrack/mod.rs | 196 ++---------------- 7 files changed, 147 insertions(+), 341 deletions(-) diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 9fa37561..79597357 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -37,7 +37,12 @@ pub fn parse_event(data: &[u8]) -> Option { stack_hash: event.data.alloc.stack_hash, }, ), - EVENT_TYPE_FREE => (event.data.free.addr, MemtrackEventKind::Free), + EVENT_TYPE_FREE => ( + event.data.free.addr, + MemtrackEventKind::Free { + stack_hash: event.data.free.stack_hash, + }, + ), EVENT_TYPE_CALLOC => ( event.data.alloc.addr, MemtrackEventKind::Calloc { diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index e580276f..b9e99d2e 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -10,6 +10,6 @@ mod tracker; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; -pub use stacks::config::StackCaptureConfig; +pub use stacks::config::{DEFAULT_STACK_COPY_SIZE, clamp_copy_size}; pub use stacks::counters::StackCaptureStats; pub use tracker::Tracker; diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs index 59ea677a..dd99b8c7 100644 --- a/crates/memtrack/src/ebpf/stacks/config.rs +++ b/crates/memtrack/src/ebpf/stacks/config.rs @@ -1,66 +1,36 @@ +use crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY; use crate::prelude::*; -pub struct StackCaptureConfig { - pub copy_size: u32, - /// Frame-pointer walk slots. Exhausting them costs the fallback chain for - /// stacks beyond the limit, never an allocation event. - pub stack_trace_capacity: u32, -} - -impl StackCaptureConfig { - pub const DEFAULT_COPY_SIZE: u32 = 8192; - pub const DEFAULT_STACK_TRACE_CAPACITY: u32 = 16384; +pub const DEFAULT_STACK_COPY_SIZE: u32 = 8192; - /// Returns `None` only when stack capture was explicitly disabled with - /// `CODSPEED_MEMTRACK_CAPTURE_STACKS=0`; capture is on by default. - pub fn from_env() -> Option { - if std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS") - .ok() - .as_deref() - == Some("0") - { - return None; - } +/// The per-allocation stack copy budget, or `None` when capture was explicitly +/// disabled with `CODSPEED_MEMTRACK_CAPTURE_STACKS=0`. Capture is on by default. +pub fn stack_copy_size_from_env() -> Option { + if std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref() == Ok("0") { + return None; + } - let copy_size = match std::env::var("CODSPEED_MEMTRACK_STACK_COPY_SIZE") { - Ok(value) => match value.parse::() { - Ok(size) => size, - Err(error) => { - warn!( - "Invalid CODSPEED_MEMTRACK_STACK_COPY_SIZE {value:?}: {error}; using default" - ); - Self::DEFAULT_COPY_SIZE - } - }, - Err(std::env::VarError::NotPresent) => Self::DEFAULT_COPY_SIZE, + let copy_size = match std::env::var("CODSPEED_MEMTRACK_STACK_COPY_SIZE") { + Ok(value) => match value.parse::() { + Ok(size) => size, Err(error) => { - warn!("Invalid CODSPEED_MEMTRACK_STACK_COPY_SIZE: {error}; using default"); - Self::DEFAULT_COPY_SIZE + warn!( + "Invalid CODSPEED_MEMTRACK_STACK_COPY_SIZE {value:?}: {error}; using default" + ); + DEFAULT_STACK_COPY_SIZE } - }; - Some(Self { - copy_size: clamp_copy_size(copy_size), - stack_trace_capacity: Self::DEFAULT_STACK_TRACE_CAPACITY, - }) - } + }, + Err(_) => DEFAULT_STACK_COPY_SIZE, + }; - pub fn with_copy_size(copy_size: u32) -> Self { - Self { - copy_size: clamp_copy_size(copy_size), - stack_trace_capacity: Self::DEFAULT_STACK_TRACE_CAPACITY, - } - } + Some(clamp_copy_size(copy_size)) } /// The kernel copies whole chunks, so a budget that is not a multiple of one /// would hash bytes it never emits. -fn clamp_copy_size(copy_size: u32) -> u32 { +pub fn clamp_copy_size(copy_size: u32) -> u32 { const CHUNK: u32 = 512; - let rounded = copy_size / CHUNK * CHUNK; - rounded.clamp( - CHUNK, - crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY, - ) + (copy_size / CHUNK * CHUNK).clamp(CHUNK, MEMTRACK_MAX_STACK_COPY) } #[cfg(test)] diff --git a/crates/memtrack/src/ebpf/stacks/counters.rs b/crates/memtrack/src/ebpf/stacks/counters.rs index a4b5ce70..6e6d76b1 100644 --- a/crates/memtrack/src/ebpf/stacks/counters.rs +++ b/crates/memtrack/src/ebpf/stacks/counters.rs @@ -1,3 +1,4 @@ +use crate::ebpf::events::bindings::*; use crate::prelude::*; #[derive(Debug, Clone, Copy, Default, serde::Serialize)] @@ -14,49 +15,24 @@ pub struct StackCaptureStats { impl StackCaptureStats { pub fn read(map: &impl libbpf_rs::MapCore) -> Result { Ok(Self { - copy_failed: slot( - map, - crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_COPY_FAILED, - "copy_failed", - )?, - hash_map_full: slot( - map, - crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_HASH_MAP_FULL, - "hash_map_full", - )?, - stackid_failed: slot( - map, - crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_STACKID_FAILED, - "stackid_failed", - )?, - truncated: slot( - map, - crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_TRUNCATED, - "truncated", - )?, - ring_full: slot( - map, - crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_RING_FULL, - "ring_full", - )?, - preempted: slot( - map, - crate::ebpf::events::bindings::MEMTRACK_STACK_COUNTER_PREEMPTED, - "preempted", - )?, + copy_failed: slot(map, MEMTRACK_STACK_COUNTER_COPY_FAILED)?, + hash_map_full: slot(map, MEMTRACK_STACK_COUNTER_HASH_MAP_FULL)?, + stackid_failed: slot(map, MEMTRACK_STACK_COUNTER_STACKID_FAILED)?, + truncated: slot(map, MEMTRACK_STACK_COUNTER_TRUNCATED)?, + ring_full: slot(map, MEMTRACK_STACK_COUNTER_RING_FULL)?, + preempted: slot(map, MEMTRACK_STACK_COUNTER_PREEMPTED)?, }) } } -fn slot(map: &impl libbpf_rs::MapCore, index: u32, name: &str) -> Result { - let key = index.to_ne_bytes(); +fn slot(map: &impl libbpf_rs::MapCore, index: u32) -> Result { let value = map - .lookup(&key, libbpf_rs::MapFlags::ANY) - .with_context(|| format!("failed to read {name} counter"))? - .ok_or_else(|| anyhow!("{name} counter slot {index} missing"))?; + .lookup(&index.to_ne_bytes(), libbpf_rs::MapFlags::ANY) + .with_context(|| format!("failed to read stack counter {index}"))? + .ok_or_else(|| anyhow!("stack counter slot {index} missing"))?; let bytes: [u8; 8] = value .as_slice() .try_into() - .map_err(|_| anyhow!("{name} counter value has unexpected size"))?; + .map_err(|_| anyhow!("stack counter {index} has unexpected size"))?; Ok(u64::from_ne_bytes(bytes)) } diff --git a/crates/memtrack/src/ebpf/stacks/events.rs b/crates/memtrack/src/ebpf/stacks/events.rs index 58125636..73bfb473 100644 --- a/crates/memtrack/src/ebpf/stacks/events.rs +++ b/crates/memtrack/src/ebpf/stacks/events.rs @@ -1,49 +1,42 @@ +use crate::ebpf::events::bindings::stack_header; use crate::prelude::*; +use libbpf_rs::MapCore; use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; -pub fn parse_stack_definition(data: &[u8]) -> Option<(MemtrackEvent, i64)> { - let header_len = std::mem::size_of::(); - if data.len() < header_len { +/// Decode one stack record from the ring buffer, returning it alongside the +/// `bpf_get_stackid()` result its frame-pointer chain is stored under. +pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { + let header_len = std::mem::size_of::(); + // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. + let header: stack_header = if data.len() >= header_len { + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } + } else { warn!( - "malformed stack definition record: got {} bytes, need at least {}", - data.len(), - header_len - ); - return None; - } - - // SAFETY: The length was checked, and the layout is the bindgen-generated C ABI struct. - let header: crate::ebpf::events::bindings::stack_def_header = - unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; - let copy_len = header.copy_len as usize; - let Some(record_len) = header_len.checked_add(copy_len) else { - warn!( - "malformed stack definition record: got {} bytes, header {} + copy_len {} overflows", - data.len(), - header_len, - copy_len + "malformed stack record: {} bytes, need at least {header_len}", + data.len() ); return None; }; + + let record_len = header_len + header.copy_len as usize; if data.len() < record_len { warn!( - "malformed stack definition record: got {} bytes, need {}", - data.len(), - record_len + "malformed stack record: {} bytes, need {record_len}", + data.len() ); return None; } - let bytes = data[header_len..record_len].to_vec(); + let event = MemtrackEvent { pid: header.pid as i32, tid: header.tid as i32, timestamp: header.timestamp, addr: 0, - kind: MemtrackEventKind::StackDefinition { + kind: MemtrackEventKind::Stack { hash: header.hash, sp: header.sp, regs: header.regs.reg.to_vec(), - bytes, + bytes: data[header_len..record_len].to_vec(), fp_chain: Vec::new(), truncated: header.truncated != 0, }, @@ -52,17 +45,41 @@ pub fn parse_stack_definition(data: &[u8]) -> Option<(MemtrackEvent, i64)> { Some((event, header.stackid)) } +/// The frame-pointer walk recorded under `stackid`, innermost frame first. +/// Best effort: a missing chain costs the fallback for one stack, not the run. +pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() +} + #[cfg(test)] mod tests { use super::*; - use crate::ebpf::events::bindings; + use crate::ebpf::events::bindings::stack_regs; - fn encode(header: bindings::stack_def_header, payload: &[u8]) -> Vec { + fn encode(header: stack_header, payload: &[u8]) -> Vec { // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. let header_bytes = unsafe { std::slice::from_raw_parts( - (&header as *const bindings::stack_def_header).cast::(), - std::mem::size_of::(), + (&header as *const stack_header).cast::(), + std::mem::size_of::(), ) }; let mut data = header_bytes.to_vec(); @@ -70,70 +87,63 @@ mod tests { data } - #[test] - fn well_formed_record_round_trips_every_field() { - let regs = std::array::from_fn(|index| 0x1000 + index as u64); - let header = bindings::stack_def_header { + fn header(copy_len: u32) -> stack_header { + stack_header { hash: 0x0123_4567_89ab_cdef, timestamp: 987_654_321, stackid: -17, sp: 0x7fff_1234_5000, pid: 41, tid: 42, - copy_len: 5, + copy_len, truncated: 1, _pad: [0; 3], - regs: bindings::stack_regs { reg: regs }, - }; + regs: stack_regs { + reg: std::array::from_fn(|index| 0x1000 + index as u64), + }, + } + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let header = header(5); let payload = [1, 2, 3, 4, 5]; - let (event, stackid) = parse_stack_definition(&encode(header, &payload)).unwrap(); + let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); assert_eq!(event.pid, 41); assert_eq!(event.tid, 42); assert_eq!(event.timestamp, 987_654_321); assert_eq!(event.addr, 0); assert_eq!(stackid, -17); - let MemtrackEventKind::StackDefinition { + let MemtrackEventKind::Stack { hash, sp, - regs: parsed_regs, + regs, bytes, fp_chain, truncated, } = event.kind else { - panic!("expected StackDefinition event"); + panic!("expected Stack event"); }; assert_eq!(hash, header.hash); assert_eq!(sp, header.sp); - assert_eq!(parsed_regs, regs.to_vec()); + assert_eq!(regs, header.regs.reg.to_vec()); assert_eq!(bytes, payload); assert!(fp_chain.is_empty()); assert!(truncated); } + #[test] fn truncated_buffer_returns_none() { - let data = vec![0; std::mem::size_of::() - 1]; - assert!(parse_stack_definition(&data).is_none()); + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack(&data).is_none()); } #[test] fn missing_payload_returns_none() { - let header = bindings::stack_def_header { - hash: 1, - timestamp: 100, - stackid: 2, - sp: 3, - pid: 4, - tid: 5, - copy_len: 4, - truncated: 0, - _pad: [0; 3], - regs: bindings::stack_regs { reg: [0; 33] }, - }; - let data = encode(header, &[1, 2, 3]); - assert!(parse_stack_definition(&data).is_none()); + assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); } } diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index a6c610e8..3a432866 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -14,14 +14,24 @@ fn generate_events(n: usize) -> Vec { for _ in 0..n { let size = rng.gen_range(8..8192); let kind = match rng.gen_range(0..10) { - 0 => MemtrackEventKind::Malloc { size }, - 1 => MemtrackEventKind::Free, + 0 => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, + 1 => MemtrackEventKind::Free { stack_hash: 0 }, 2 => MemtrackEventKind::Realloc { old_addr: Some(rng.r#gen()), size, + stack_hash: 0, + }, + 3 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, + 4 => MemtrackEventKind::AlignedAlloc { + size, + stack_hash: 0, }, - 3 => MemtrackEventKind::Calloc { size }, - 4 => MemtrackEventKind::AlignedAlloc { size }, 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, @@ -90,12 +100,18 @@ fn generate_realistic_events(n: usize) -> Vec { addr }); let kind = match rng.gen_range(0..20) { - 0 => MemtrackEventKind::Calloc { size }, + 0 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, 1 => MemtrackEventKind::Mmap { size }, - _ => MemtrackEventKind::Malloc { size }, + _ => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, }; - if let MemtrackEventKind::Mmap { size } = kind { - live_mmap.push((addr, size)); + if let MemtrackEventKind::Mmap { size } = &kind { + live_mmap.push((addr, *size)); } else { live_heap.push(addr); } @@ -105,7 +121,7 @@ fn generate_realistic_events(n: usize) -> Vec { if idx < live_heap.len() { let addr = live_heap.swap_remove(idx); free_list.push(addr); - (addr, MemtrackEventKind::Free) + (addr, MemtrackEventKind::Free { stack_hash: 0 }) } else { let (addr, size) = live_mmap.swap_remove(idx - live_heap.len()); free_list.push(addr); @@ -127,6 +143,7 @@ fn generate_realistic_events(n: usize) -> Vec { MemtrackEventKind::Realloc { old_addr: Some(old_addr), size, + stack_hash: 0, }, ) }; @@ -150,7 +167,7 @@ fn encode_events_realistic(bencher: Bencher, n_workers: usize) { bencher.bench_local(|| { let mut output = Vec::new(); - encode_events(events.iter().copied(), &mut output, n_workers).unwrap(); + encode_events(events.iter().cloned(), &mut output, n_workers).unwrap(); output }); } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index 0778bd76..496a8764 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -59,7 +59,10 @@ pub enum MemtrackEventKind { #[serde(default, skip_serializing_if = "is_zero")] stack_hash: u64, }, - Free, + Free { + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, + }, Realloc { #[serde(default, skip_serializing_if = "Option::is_none")] old_addr: Option, @@ -100,9 +103,9 @@ pub enum MemtrackEventKind { delta: i64, }, - /// Raw stack for one distinct `stack_hash`, emitted once per hash. Allocation events - /// reference it by hash. Unwinding happens off-box. - StackDefinition { + /// Raw stack for one distinct `stack_hash`, emitted once per hash; the + /// allocation events carrying that hash reference it. Unwound off-box. + Stack { hash: u64, /// User stack pointer the copy starts at. sp: u64, @@ -116,26 +119,12 @@ pub enum MemtrackEventKind { /// The copy filled its budget, so stack above it was not captured. truncated: bool, }, - - /// One executable file mapping observed in a tracked process (event pid = the process). - /// The runner resolves `path` to symbols + unwind data after the run. - ModuleMapping { - /// Runtime AVMA range and file offset, straight from /proc//maps. - start: u64, - end: u64, - offset: u64, - path: String, - }, - - /// Process name (/proc//comm) for the event's pid, emitted when first observed. - Comm { - name: String, - }, } fn is_zero(value: &u64) -> bool { *value == 0 } + pub struct MemtrackEventStream { deserializer: rmp_serde::Deserializer>, } @@ -173,7 +162,7 @@ mod tests { tid: 12, timestamp: 200, addr: 0x20, - kind: MemtrackEventKind::Free, + kind: MemtrackEventKind::Free { stack_hash: 0 }, }, MemtrackEvent { pid: 1, @@ -221,7 +210,8 @@ mod tests { size: 7, stack_hash: 0xCAFE_BABE, }, - MemtrackEventKind::Free, + MemtrackEventKind::Free { stack_hash: 0 }, + MemtrackEventKind::Free { stack_hash: 0xFEED }, MemtrackEventKind::Realloc { old_addr: Some(0x1000), size: 42, @@ -243,7 +233,7 @@ mod tests { MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, - MemtrackEventKind::StackDefinition { + MemtrackEventKind::Stack { hash: 0xDEAD_BEEF, sp: 0x7FFF_0000, regs: vec![0; 33], @@ -251,15 +241,6 @@ mod tests { fp_chain: vec![0x1000, 0x2000], truncated: false, }, - MemtrackEventKind::ModuleMapping { - start: 0x400000, - end: 0x401000, - offset: 0, - path: "/usr/bin/app".to_string(), - }, - MemtrackEventKind::Comm { - name: "app".to_string(), - }, ]; for kind in kinds { @@ -285,159 +266,6 @@ mod tests { } } - #[test] - fn zero_stack_hash_serializes_byte_identically_to_legacy_shape() { - #[derive(serde::Serialize)] - #[serde(tag = "type")] - enum LegacyKind { - Malloc { - size: u64, - }, - Calloc { - size: u64, - }, - AlignedAlloc { - size: u64, - }, - Realloc { - #[serde(default, skip_serializing_if = "Option::is_none")] - old_addr: Option, - size: u64, - }, - } - - #[derive(serde::Serialize)] - struct LegacyEvent { - pid: libc::pid_t, - tid: libc::pid_t, - timestamp: u64, - addr: u64, - #[serde(flatten)] - kind: LegacyKind, - } - - let cases = [ - ( - MemtrackEventKind::Malloc { - size: 128, - stack_hash: 0, - }, - LegacyKind::Malloc { size: 128 }, - ), - ( - MemtrackEventKind::Calloc { - size: 256, - stack_hash: 0, - }, - LegacyKind::Calloc { size: 256 }, - ), - ( - MemtrackEventKind::AlignedAlloc { - size: 512, - stack_hash: 0, - }, - LegacyKind::AlignedAlloc { size: 512 }, - ), - ( - MemtrackEventKind::Realloc { - old_addr: Some(0x1000), - size: 1024, - stack_hash: 0, - }, - LegacyKind::Realloc { - old_addr: Some(0x1000), - size: 1024, - }, - ), - ( - MemtrackEventKind::Realloc { - old_addr: None, - size: 1024, - stack_hash: 0, - }, - LegacyKind::Realloc { - old_addr: None, - size: 1024, - }, - ), - ]; - - for (kind, legacy_kind) in cases { - let event = MemtrackEvent { - pid: 123, - tid: 456, - timestamp: 789, - addr: 0xABCD, - kind, - }; - let legacy = LegacyEvent { - pid: 123, - tid: 456, - timestamp: 789, - addr: 0xABCD, - kind: legacy_kind, - }; - - assert_eq!( - rmp_serde::to_vec(&event).unwrap(), - rmp_serde::to_vec(&legacy).unwrap() - ); - } - } - - #[test] - fn roundtrip_extended_events() -> anyhow::Result<()> { - let events = vec![ - MemtrackEvent { - pid: 42, - tid: 42, - timestamp: 1000, - addr: 0, - kind: MemtrackEventKind::StackDefinition { - hash: 0xFEED_FACE_CAFE_BEEF, - sp: 0x7FFF_FFFF_0000, - regs: (0..33).map(|r| r * 0x100).collect(), - bytes: vec![0xDE, 0xAD, 0xBE, 0xEF, 0xAA, 0xBB, 0xCC, 0xDD], - fp_chain: vec![0x400500, 0x400600, 0x400700], - truncated: true, - }, - }, - MemtrackEvent { - pid: 42, - tid: 42, - timestamp: 1001, - addr: 0, - kind: MemtrackEventKind::ModuleMapping { - start: 0x7FFF_0000, - end: 0x7FFF_1000, - offset: 0x2000, - path: "/lib/x86_64-linux-gnu/libc.so.6".to_string(), - }, - }, - MemtrackEvent { - pid: 42, - tid: 42, - timestamp: 1002, - addr: 0, - kind: MemtrackEventKind::Comm { - name: "worker-thread".to_string(), - }, - }, - ]; - - let artifact = MemtrackArtifact { - events: events.clone(), - }; - let mut buf = Vec::new(); - artifact.encode_to_writer(&mut buf)?; - - let stream = MemtrackArtifact::decode_streamed(Cursor::new(buf))?; - let collected: Vec<_> = stream.collect(); - assert_eq!(collected, events); - - Ok(()) - } - #[test] fn concatenated_frames_decode_in_order() -> anyhow::Result<()> { let events: Vec<_> = (0..2500) From ade4a49e90025d09ef3acfea9def94bcb2fdb82e Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 17:06:03 +0200 Subject: [PATCH 15/16] fixup! feat(memtrack): enable stack capture through the tracker --- crates/memtrack/src/ebpf/memtrack/maps.rs | 30 --------- crates/memtrack/src/ebpf/memtrack/mod.rs | 80 +++++++---------------- crates/memtrack/src/ebpf/tracker.rs | 60 ++++++++--------- 3 files changed, 53 insertions(+), 117 deletions(-) diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index 939f1bf9..0a60f270 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -73,36 +73,6 @@ impl MemtrackBpf { StackCaptureStats::read(with_skel!(self, skel => &skel.maps.stack_counters)) } - /// The frame-pointer walk recorded under `stackid`, innermost frame first. - /// Empty when the id is negative (the walk failed) or the entry was evicted. - /// Reading it is best effort: a missing chain costs the fallback for one - /// stack, not the run. - pub fn fp_chain(&self, stackid: i64) -> Vec { - let Ok(key) = u32::try_from(stackid) else { - return Vec::new(); - }; - - let entry = with_skel!(self, skel => skel - .maps - .stack_traces - .lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY)); - let value = match entry { - Ok(Some(value)) => value, - Ok(None) => return Vec::new(), - Err(error) => { - warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); - return Vec::new(); - } - }; - - // The map value is a fixed-depth array zero-padded past the last frame. - value - .chunks_exact(8) - .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) - .take_while(|&address| address != 0) - .collect() - } - pub fn ownership_maps(&self) -> Result { let owner_by_mm = entries(with_skel!(self, skel => &skel.maps.owner_by_mm))?; let mm_by_pid = entries(with_skel!(self, skel => &skel.maps.mm_by_pid))?; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 13da1f72..524478ab 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -7,7 +7,6 @@ use std::mem::MaybeUninit; use std::path::Path; use crate::ebpf::poller::RingBufferPoller; -use crate::ebpf::stacks::config::StackCaptureConfig; mod token { include!(concat!(env!("OUT_DIR"), "/memtrack_token.skel.rs")); @@ -124,24 +123,24 @@ pub struct MemtrackBpf { impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool, stacks: Option<&StackCaptureConfig>) -> Result { + pub fn new_with_rmap(track_rmap: bool, stack_copy_size: Option) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap, stacks) + Self::with_variant(variant, track_rmap, stack_copy_size) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. /// - /// `stacks` turns on allocation stack capture and sizes its maps. + /// `stack_copy_size` turns on allocation stack capture. pub fn with_variant( variant: BpfVariant, track_rmap: bool, - stacks: Option<&StackCaptureConfig>, + stack_copy_size: Option, ) -> Result { let page_shift = page_shift()?; let rmap = if track_rmap { @@ -171,25 +170,19 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } - if let Some(stacks) = stacks { + if let Some(copy_size) = stack_copy_size { rodata.capture_stacks_enabled = 1; - rodata.stack_copy_size = stacks.copy_size; + rodata.stack_copy_size = copy_size; } } // Avoid reserving the stack maps when capture is disabled. A // ring buffer's size must stay a power-of-two page count. - match stacks { - Some(stacks) => open_skel - .maps - .stack_traces - .set_max_entries(stacks.stack_trace_capacity)?, - None => { - open_skel.maps.stack_defs.set_max_entries(4096)?; - open_skel.maps.stack_traces.set_max_entries(1)?; - open_skel.maps.seen_stack_hashes.set_max_entries(1)?; - open_skel.maps.pending_stack_hash.set_max_entries(1)?; - } + if stack_copy_size.is_none() { + open_skel.maps.stacks.set_max_entries(4096)?; + open_skel.maps.stack_traces.set_max_entries(1)?; + open_skel.maps.seen_stack_hashes.set_max_entries(1)?; + open_skel.maps.pending_stack_hash.set_max_entries(1)?; } // Autoload is decided before load(), so fentries whose targets @@ -240,10 +233,6 @@ impl MemtrackBpf { /// Poll the allocation-event ring buffer into `tx`. The returned poller /// keeps the pipeline alive; events stop flowing when it is dropped. - /// - /// `stack_tx` receives the stack identity carried by each allocation event. - /// It rides along on this ring rather than on its own because the identity - /// lives inside the event record. pub fn poll_events_with_channel( &self, poll_interval_ms: u64, @@ -257,55 +246,32 @@ impl MemtrackBpf { )) } - /// Poll the stack-definition ring buffer into `tx`. - pub(crate) fn poll_stack_definitions( + /// Poll the stack-record ring buffer into `tx`. + pub(crate) fn poll_stacks( &self, poll_interval_ms: u64, tx: std::sync::mpsc::Sender, ) -> Result { - use libbpf_rs::MapCore; + use crate::ebpf::stacks::events; + use runner_shared::artifacts::MemtrackEventKind; + + // The poller outlives this borrow of the skeleton, so the chain lookup + // needs an owned handle rather than a reference to the skeleton map. let stack_traces = with_skel!(self, skel => { libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) .context("Failed to create handle for stack_traces map")? }); - let parse = move |data: &[u8]| -> Option { - let (mut event, stackid) = crate::ebpf::stacks::events::parse_stack_definition(data)?; - let fp_chain = if stackid < 0 { - Vec::new() - } else if let Ok(key) = u32::try_from(stackid) { - match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { - Ok(Some(value)) => value - .chunks_exact(8) - .map(|word| { - u64::from_ne_bytes( - word.try_into().expect("chunks_exact yields 8 bytes"), - ) - }) - .take_while(|&address| address != 0) - .collect(), - Ok(None) => Vec::new(), - Err(error) => { - warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); - Vec::new() - } - } - } else { - Vec::new() - }; - - if let runner_shared::artifacts::MemtrackEventKind::StackDefinition { - fp_chain: ref mut chain, - .. - } = event.kind - { - *chain = fp_chain; + let parse = move |data: &[u8]| { + let (mut event, stackid) = events::parse_stack(data)?; + if let MemtrackEventKind::Stack { fp_chain, .. } = &mut event.kind { + *fp_chain = events::fp_chain(&stack_traces, stackid); } Some(event) }; with_skel!(self, skel => RingBufferPoller::new( - &skel.maps.stack_defs, + &skel.maps.stacks, parse, tx, poll_interval_ms, diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index db067799..5689abdc 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,6 +1,6 @@ use crate::ebpf::attach_worker::AttachWorker; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; -use crate::ebpf::stacks::config::StackCaptureConfig; +use crate::ebpf::stacks::config::{clamp_copy_size, stack_copy_size_from_env}; use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; @@ -9,32 +9,37 @@ use parking_lot::Mutex; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; pub struct Tracker { bpf: Arc>, worker: Mutex>, allocators: bool, - stack_config: Mutex>, - stack_capture_enabled: bool, + /// The dedup gate spans the whole BPF object, so a second session would + /// reference stack records the first one already consumed. + stacks_polled: Option, } impl Tracker { /// Create a new tracker. The exec-mapping watcher discovers and attaches /// allocator probes as the tracked process tree maps executable files. pub fn new() -> Result { - Self::with_stack_config(StackCaptureConfig::from_env()) + Self::with_stack_capture_size(stack_copy_size_from_env()) } /// Like [`Tracker::new`], with allocation stack capture forced on. - pub fn new_with_stack_capture(config: StackCaptureConfig) -> Result { - Self::with_stack_config(Some(config)) + pub fn new_with_stack_capture(copy_size: u32) -> Result { + Self::with_stack_capture_size(Some(clamp_copy_size(copy_size))) } - fn with_stack_config(stacks: Option) -> Result { + fn with_stack_capture_size(copy_size: Option) -> Result { let track_rmap = Self::track_rmap_from_env(); - let bpf = MemtrackBpf::new_with_rmap(track_rmap, stacks.as_ref())?; - Self::build(bpf, true, stacks) + Self::build( + MemtrackBpf::new_with_rmap(track_rmap, copy_size)?, + true, + copy_size.is_some(), + ) } /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of @@ -44,7 +49,7 @@ impl Tracker { Self::build( MemtrackBpf::with_variant(variant, track_rmap, None)?, true, - None, + false, ) } @@ -56,14 +61,10 @@ impl Tracker { /// rmap fentries) without allocator probes; no exec-mapping watcher or /// attach worker runs. pub fn new_without_allocators_with_rmap(track_rmap: bool) -> Result { - Self::build(MemtrackBpf::new_with_rmap(track_rmap, None)?, false, None) + Self::build(MemtrackBpf::new_with_rmap(track_rmap, None)?, false, false) } - fn build( - mut bpf: MemtrackBpf, - allocators: bool, - stacks: Option, - ) -> Result { + fn build(mut bpf: MemtrackBpf, allocators: bool, capture_stacks: bool) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; @@ -78,13 +79,11 @@ impl Tracker { None }; - let stack_capture_enabled = stacks.is_some(); Ok(Self { bpf, worker: Mutex::new(worker), allocators, - stack_config: Mutex::new(stacks), - stack_capture_enabled, + stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), }) } @@ -97,10 +96,13 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { - if self.stack_capture_enabled && self.stack_config.lock().is_none() { - bail!("stack capture supports a single spawned command per tracker"); - } - let stack_capture_on = self.stack_config.lock().take().is_some(); + let capture_stacks = match &self.stacks_polled { + Some(polled) if polled.swap(true, Ordering::Relaxed) => { + bail!("stack capture supports a single spawned command per tracker") + } + Some(_) => true, + None => false, + }; let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { @@ -110,7 +112,6 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - let (tx, rx) = mpsc::channel(); match self.worker.lock().as_ref() { Some(worker) => worker.set_root_pid(pid), // No watcher to arm means exec mappings would be missed. @@ -118,14 +119,13 @@ impl Tracker { None => {} } + let (tx, rx) = mpsc::channel(); let (poller, stack_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; - let stack_poller = if stack_capture_on { - Some(bpf.poll_stack_definitions(10, tx.clone())?) - } else { - None - }; + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; (bpf.poll_events_with_channel(10, tx)?, stack_poller) }; resume(pid)?; @@ -151,7 +151,7 @@ impl Tracker { self.bpf.lock().dropped_events_count() } - /// Read the stack capture counters from BPF map. + /// Per-cause counts of stack captures that were skipped or truncated. pub fn stack_capture_stats(&self) -> Result { self.bpf.lock().stack_capture_stats() } From 83de75efe176949843c25bb4df061072c76be8f2 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 28 Aug 2026 17:06:04 +0200 Subject: [PATCH 16/16] fixup! test(memtrack): cover allocation stack capture --- .github/workflows/ci.yml | 2 +- crates/memtrack/tests/dlopen_tests.rs | 3 +- crates/memtrack/tests/shared.rs | 26 +- crates/memtrack/tests/stack_tests.rs | 358 ++++++++------------------ 4 files changed, 122 insertions(+), 267 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fe75ea..3265a919 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests, stack_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/crates/memtrack/tests/dlopen_tests.rs b/crates/memtrack/tests/dlopen_tests.rs index ec243ae2..dfca5044 100644 --- a/crates/memtrack/tests/dlopen_tests.rs +++ b/crates/memtrack/tests/dlopen_tests.rs @@ -78,7 +78,8 @@ fn test_dlopen_allocator() -> Result<(), Box> { let free_count = events .iter() .filter(|e| { - matches!(e.kind, MemtrackEventKind::Free) && malloc_addrs.contains(&e.addr) + matches!(e.kind, MemtrackEventKind::Free { .. }) + && malloc_addrs.contains(&e.addr) }) .count(); diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 903947fb..d307b626 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -25,7 +25,7 @@ macro_rules! assert_events_snapshot { let formatted_events: Vec = $events .iter() .filter(|e| { - // RSS, lifecycle, stack, and mapping events are asserted by dedicated tests, not snapshots. + // RSS, lifecycle and stack events are asserted by dedicated tests, not snapshots. !matches!( e.kind, MemtrackEventKind::Rss { .. } @@ -33,9 +33,7 @@ macro_rules! assert_events_snapshot { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit - | MemtrackEventKind::StackDefinition { .. } - | MemtrackEventKind::ModuleMapping { .. } - | MemtrackEventKind::Comm { .. } + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -91,11 +89,11 @@ macro_rules! assert_events_with_marker_for_each_variant { }; } -/// An event's kind and size, without the addresses that differ between runs of -/// the same workload. `Realloc` needs spelling out since its `Debug` includes the -/// old address. +/// An event's kind and size, without the addresses or stack identities that +/// differ between runs of the same workload. pub fn describe_kind(kind: &MemtrackEventKind) -> String { match kind { + MemtrackEventKind::Free { .. } => "Free".to_string(), MemtrackEventKind::Malloc { size, .. } => format!("Malloc {{ size: {size} }}"), MemtrackEventKind::Calloc { size, .. } => format!("Calloc {{ size: {size} }}"), MemtrackEventKind::AlignedAlloc { size, .. } => format!("AlignedAlloc {{ size: {size} }}"), @@ -128,9 +126,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit - | MemtrackEventKind::StackDefinition { .. } - | MemtrackEventKind::ModuleMapping { .. } - | MemtrackEventKind::Comm { .. } + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -230,12 +226,10 @@ pub fn track_command_with_rmap_maps( let maps = tracker.ownership_maps()?; Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } + /// Track a command with allocation stack capture enabled, returning its events. -pub fn track_command_with_stacks( - command: Command, - config: memtrack::StackCaptureConfig, -) -> TrackResult { - track_command_with_tracker(command, Tracker::new_with_stack_capture(config)?) +pub fn track_command_with_stacks(command: Command, copy_size: u32) -> TrackResult { + track_command_with_tracker(command, Tracker::new_with_stack_capture(copy_size)?) } /// Track a command with rmap hooks and snapshot the ownership maps at a @@ -282,7 +276,7 @@ fn event_profile(events: &[Event]) -> EventProfile { if !matches!( event.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs index 6cb63a5f..8cce5424 100644 --- a/crates/memtrack/tests/stack_tests.rs +++ b/crates/memtrack/tests/stack_tests.rs @@ -1,104 +1,117 @@ #[macro_use] mod shared; -use runner_shared::artifacts::MemtrackEventKind; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; use std::collections::HashSet; use std::process::Command; use tempfile::TempDir; +const COPY_SIZE: u32 = memtrack::DEFAULT_STACK_COPY_SIZE; + +fn compile_fixture( + name: &str, + temp_dir: &TempDir, +) -> Result> { + shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + name, + temp_dir.path(), + ) +} + +/// The stack identity carried by each allocation and deallocation event that has one. +fn event_hashes(events: &[MemtrackEvent]) -> Vec { + events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } + | MemtrackEventKind::Free { stack_hash } => (stack_hash != 0).then_some(stack_hash), + _ => None, + }) + .collect() +} + +fn record_hashes(events: &[MemtrackEvent]) -> HashSet { + events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Stack { hash, .. } => Some(hash), + _ => None, + }) + .collect() +} + #[test_with::env(GITHUB_ACTIONS)] #[test_log::test] fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source( - include_str!("../testdata/stack_paths.c"), - "stack_paths", - temp_dir.path(), - )?; - let copy_size = memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE; - let (events, thread_handle) = shared::track_command_with_stacks( - Command::new(&binary), - memtrack::StackCaptureConfig::with_copy_size(copy_size), - )?; + let binary = compile_fixture("stack_paths", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; - let definitions: Vec<_> = events + let records: Vec<_> = events .iter() .filter_map(|e| match &e.kind { - MemtrackEventKind::StackDefinition { + MemtrackEventKind::Stack { hash, sp, regs, bytes, - fp_chain, truncated, - } => Some((e, *hash, *sp, regs, bytes, fp_chain, *truncated)), + .. + } => Some((*hash, *sp, regs, bytes, *truncated)), _ => None, }) .collect(); assert!( - definitions.len() >= 2, - "expected at least two stack definitions, got {} ({} events)", - definitions.len(), + records.len() >= 2, + "expected at least two stack records, got {} ({} events)", + records.len(), events.len() ); - let hashes: HashSet = definitions.iter().map(|(_, hash, ..)| *hash).collect(); + let hashes = record_hashes(&events); assert_eq!( hashes.len(), - definitions.len(), - "stack definitions must be deduplicated by unique hash" - ); + records.len(), + "stack records must be deduplicated by unique hash" + ); + + for (hash, sp, regs, bytes, truncated) in &records { + assert_ne!(*sp, 0, "record {hash:#x} has no stack pointer"); + assert_eq!(regs.len(), 33, "record {hash:#x} must carry 33 registers"); + assert!( + !bytes.is_empty() && bytes.len() % 512 == 0 && bytes.len() <= COPY_SIZE as usize, + "record {hash:#x} must hold whole 512-byte chunks within the budget, got {}", + bytes.len() + ); + assert_eq!( + *truncated, + bytes.len() == COPY_SIZE as usize, + "record {hash:#x} may only be flagged truncated when it filled the budget" + ); + } + let carried = event_hashes(&events); assert!( - definitions - .iter() - .all(|(_, _, _, _, bytes, _, _)| !bytes.is_empty()), - "every stack definition must contain copied stack bytes" - ); - assert!( - definitions.iter().all(|(_, _, sp, ..)| *sp != 0), - "every stack definition must include a nonzero stack pointer" + !carried.is_empty(), + "expected events carrying a captured stack hash" ); assert!( - definitions - .iter() - .all(|(_, _, _, regs, ..)| regs.len() == 33), - "every stack definition must include 33 registers on x86_64" - ); - assert!( - definitions.iter().all(|(_, _, _, _, bytes, _, truncated)| { - bytes.len() <= copy_size as usize - && bytes.len() % 512 == 0 - && *truncated == (bytes.len() == copy_size as usize) - }), - "stack copies must be whole 512-byte chunks within copy_size and flagged truncated only when full" + carried.iter().all(|hash| hashes.contains(hash)), + "every non-zero stack_hash must have a matching stack record" ); - let alloc_hashes: Vec = events - .iter() - .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { stack_hash, .. } - | MemtrackEventKind::Calloc { stack_hash, .. } - | MemtrackEventKind::AlignedAlloc { stack_hash, .. } - | MemtrackEventKind::Realloc { stack_hash, .. } => { - if stack_hash != 0 { - Some(stack_hash) - } else { - None - } - } - _ => None, - }) - .collect(); - + // The fixture frees every allocation, so both sides must report identities. assert!( - !alloc_hashes.is_empty(), - "expected at least one allocation event carrying a captured stack hash" - ); - assert!( - alloc_hashes.iter().all(|hash| hashes.contains(hash)), - "every non-zero stack_hash on an alloc must have a matching StackDefinition hash" + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Free { stack_hash } if stack_hash != 0)), + "free events must carry their own stack identity" ); thread_handle @@ -111,41 +124,17 @@ fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source( - include_str!("../testdata/stack_paths.c"), - "stack_paths_dedup", - temp_dir.path(), - )?; - let (events, thread_handle) = shared::track_command_with_stacks( - Command::new(&binary), - memtrack::StackCaptureConfig::with_copy_size( - memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE, - ), - )?; - - let def_hashes: HashSet = events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::StackDefinition { hash, .. } => Some(*hash), - _ => None, - }) - .collect(); - - let alloc_with_stack_count = events - .iter() - .filter(|e| match e.kind { - MemtrackEventKind::Malloc { stack_hash, .. } - | MemtrackEventKind::Calloc { stack_hash, .. } - | MemtrackEventKind::AlignedAlloc { stack_hash, .. } - | MemtrackEventKind::Realloc { stack_hash, .. } => stack_hash != 0, - _ => false, - }) - .count(); + let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + let carried = event_hashes(&events); + let records = record_hashes(&events); assert!( - alloc_with_stack_count > def_hashes.len(), - "expected repeated call paths to deduplicate raw stacks: {alloc_with_stack_count} stack-bearing events across {} unique stacks ({} total events)", - def_hashes.len(), + carried.len() > records.len(), + "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", + carried.len(), + records.len(), events.len() ); @@ -164,37 +153,29 @@ fn dedup_collapses_repeated_call_paths() -> Result<(), Box Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source( - include_str!("../testdata/stack_paths.c"), - "stack_paths_max", - temp_dir.path(), - )?; - // Clamped down to the largest copy a stack-definition record can carry. - let (events, thread_handle) = shared::track_command_with_stacks( - Command::new(&binary), - memtrack::StackCaptureConfig::with_copy_size(u32::MAX), - )?; + let binary = compile_fixture("stack_paths_max", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), u32::MAX)?; - let definitions: Vec<_> = events + let truncated: Vec<_> = events .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::StackDefinition { + .filter_map(|e| match e.kind { + MemtrackEventKind::Stack { hash, - bytes, - truncated, + truncated: true, .. - } => Some((*hash, bytes.len(), *truncated)), + } => Some(hash), _ => None, }) .collect(); assert!( - !definitions.is_empty(), - "expected stack definitions at the maximum copy budget" + !record_hashes(&events).is_empty(), + "expected stack records at the maximum copy budget" ); assert!( - definitions.iter().all(|(_, _, truncated)| !*truncated), - "no capture can be budget-limited at the maximum budget: {definitions:?}" + truncated.is_empty(), + "no capture can be budget-limited at the maximum budget: {truncated:#x?}" ); thread_handle @@ -203,95 +184,6 @@ fn max_copy_budget_loads_and_captures_whole_stacks() -> Result<(), Box Result<(), Box> { - let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source( - include_str!("../testdata/stack_paths.c"), - "stack_paths_tiny_map", - temp_dir.path(), - )?; - - let (baseline_events, baseline_handle) = shared::track_command_with_stacks( - Command::new(&binary), - memtrack::StackCaptureConfig::with_copy_size( - memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE, - ), - )?; - baseline_handle - .join() - .expect("tracker teardown thread panicked"); - - let (events, thread_handle) = shared::track_command_with_stacks( - Command::new(&binary), - memtrack::StackCaptureConfig { - stack_trace_capacity: 1, - ..memtrack::StackCaptureConfig::with_copy_size( - memtrack::StackCaptureConfig::DEFAULT_COPY_SIZE, - ) - }, - )?; - thread_handle - .join() - .expect("tracker teardown thread panicked"); - - let baseline_allocs_with_stack = baseline_events - .iter() - .filter(|e| match e.kind { - MemtrackEventKind::Malloc { stack_hash, .. } - | MemtrackEventKind::Calloc { stack_hash, .. } - | MemtrackEventKind::AlignedAlloc { stack_hash, .. } - | MemtrackEventKind::Realloc { stack_hash, .. } => stack_hash != 0, - _ => false, - }) - .count(); - - let allocs_with_stack = events - .iter() - .filter(|e| match e.kind { - MemtrackEventKind::Malloc { stack_hash, .. } - | MemtrackEventKind::Calloc { stack_hash, .. } - | MemtrackEventKind::AlignedAlloc { stack_hash, .. } - | MemtrackEventKind::Realloc { stack_hash, .. } => stack_hash != 0, - _ => false, - }) - .count(); - - assert_eq!( - allocs_with_stack, baseline_allocs_with_stack, - "allocation totals with stack must be unaffected by frame-pointer map exhaustion" - ); - - let baseline_def_hashes: HashSet = baseline_events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::StackDefinition { hash, .. } => Some(*hash), - _ => None, - }) - .collect(); - - let def_hashes: HashSet = events - .iter() - .filter_map(|e| match &e.kind { - MemtrackEventKind::StackDefinition { hash, .. } => Some(*hash), - _ => None, - }) - .collect(); - - // Hash values are not comparable across runs: the copied bytes embed - // ASLR-randomized addresses, so only the dedup structure is deterministic. - assert_eq!( - def_hashes.len(), - baseline_def_hashes.len(), - "stack identity comes from the copied bytes, not the frame-pointer walk" - ); - - Ok(()) -} - /// Restores the capture toggle on drop so a failing assertion cannot leak the /// override into later tests (the suite runs single-threaded). struct DisableCaptureGuard; @@ -315,55 +207,23 @@ impl Drop for DisableCaptureGuard { #[test_log::test] fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { let temp_dir = TempDir::new()?; - let binary = shared::compile_c_source( - include_str!("../testdata/stack_paths.c"), - "stack_paths_disabled", - temp_dir.path(), - )?; + let binary = compile_fixture("stack_paths_disabled", &temp_dir)?; let _guard = DisableCaptureGuard::set(); let (events, thread_handle) = shared::track_binary(&binary)?; - let alloc_events: Vec<_> = events - .iter() - .filter(|e| { - matches!( - e.kind, - MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Calloc { .. } - | MemtrackEventKind::AlignedAlloc { .. } - | MemtrackEventKind::Realloc { .. } - | MemtrackEventKind::Free - ) - }) - .collect(); - assert!( - !alloc_events.is_empty(), + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Malloc { .. })), "disabled capture must still report allocation events" ); - - let stack_defs_count = events - .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::StackDefinition { .. })) - .count(); - assert_eq!( - stack_defs_count, 0, - "disabled capture must emit zero StackDefinition events" + assert!( + record_hashes(&events).is_empty(), + "disabled capture must emit zero stack records" ); - - let non_zero_stack_hashes = events - .iter() - .filter(|e| match e.kind { - MemtrackEventKind::Malloc { stack_hash, .. } - | MemtrackEventKind::Calloc { stack_hash, .. } - | MemtrackEventKind::AlignedAlloc { stack_hash, .. } - | MemtrackEventKind::Realloc { stack_hash, .. } => stack_hash != 0, - _ => false, - }) - .count(); - assert_eq!( - non_zero_stack_hashes, 0, - "disabled capture must leave stack_hash zero on allocation events" + assert!( + event_hashes(&events).is_empty(), + "disabled capture must leave stack_hash zero on every event" ); thread_handle