diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8fe75eac..3265a9194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests, stack_tests] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: diff --git a/Cargo.lock b/Cargo.lock index 9cf2031c7..1f63a302e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3615,6 +3615,7 @@ dependencies = [ "rmp", "rmp-serde", "serde", + "serde_bytes", "serde_json", "zstd", ] @@ -4061,6 +4062,16 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" diff --git a/crates/memtrack/AGENTS.md b/crates/memtrack/AGENTS.md index 0c8d86d41..819eb584c 100644 --- a/crates/memtrack/AGENTS.md +++ b/crates/memtrack/AGENTS.md @@ -20,11 +20,13 @@ Control plane: `src/ipc.rs` exposes an out-of-band `ipc-channel` protocol (`Enab Allocator discovery (`src/allocators/`): `AllocatorLib::find_all()` = dynamic (glob shared libs incl. `/nix/store/*` hints) + static-linked (scan build-dir ELF symbols) + env (`CODSPEED_MEMTRACK_BINARIES`). Each `AllocatorKind` (`Libc`/`LibCpp`/`Jemalloc`/`Mimalloc`/`Tcmalloc`) maps to best-effort attach helpers; only libc must succeed. +Mapping recorder (`src/ebpf/c/mappings.bpf.h`, `src/ebpf/mappings/`): an LSM program on `mmap_file` resolves each mapped file's path once per inode into `path_by_inode` (`bpf_path_d_path` on kernels >= 6.12, `bpf_d_path` on the sleepable hook from 5.11), and an `fentry/perf_event_mmap` program emits executable-mapping geometry (`dev`/`ino`/`file_offset`/`start`/`end`) on the `mappings` ring buffer. Userspace joins the two into a `MemtrackMappings` artifact, which the runner turns into `unwind_data`/`symbols.map` files and a `memtrack.metadata` so allocation stacks unwind off-box. `MappingSupport::detect()` gates the programs on the kernel release **and** `bpf` being in `/sys/kernel/security/lsm`; with neither available, stack capture is disabled since nothing could attribute the stacks. + > Note: the "on-demand attach" design in `.agents/docs/` (AttachWorker, `CODSPEED_MEMTRACK_ONDEMAND`, SIGSTOP/SIGCONT) is a **plan, not yet in source**. Current behavior is upfront attach + `sched_fork` auto-tracking. ## Key Directories -- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/utils/*.h` + `c/allocator.h`. +- `src/ebpf/` — BPF stack (feature-gated `ebpf`): `tracker.rs` (facade), `memtrack/` (libbpf-rs wrapper + generated skeleton, split into `mod.rs`/`macros.rs`/`maps.rs`/`allocator.rs`/`tracking.rs`), `mappings/` (records/resolve/support), `poller.rs`, `events.rs`, `c/main.bpf.c` + `c/event.h` + `c/mappings.bpf.h` + `c/utils/*.h` + `c/allocator.h`. - `src/allocators/` — allocator classification: `mod.rs`, `dynamic.rs`, `static_linked.rs`. - `tests/` — integration tests + `snapshots/` (insta). - `testdata/` — allocation fixtures: `*.c` (gcc), `alloc_cpp/` (cmkr/CMake), `alloc_rust/` + `spawn_wrapper/` (standalone Cargo workspaces). diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 9a4cc2387..8de96317f 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -9,6 +9,7 @@ BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ SEC(UPROBE_SEC) \ int uprobe_##name(struct pt_regs* ctx) { \ + stash_stack_hash(capture_stack(ctx)); \ return store_param(&name##_arg, arg_expr); \ } \ SEC(URETPROBE_SEC) \ @@ -17,6 +18,7 @@ if (!arg_ptr) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ __u64 ret_val = PT_REGS_RC(ctx); \ if (ret_val == 0) { \ return 0; \ @@ -32,6 +34,7 @@ if (arg0 == 0) { \ return 0; \ } \ + __u64 stack_hash = capture_stack(ctx); \ submit_block; \ } @@ -50,6 +53,8 @@ return 0; \ } \ \ + stash_stack_hash(capture_stack(ctx)); \ + \ struct name##_args_t args = {.arg0 = arg0_expr, .arg1 = arg1_expr}; \ \ bpf_map_update_elem(&name##_args, &tid, &args, BPF_ANY); \ @@ -63,6 +68,7 @@ if (!args) { \ return 0; \ } \ + __u64 stack_hash = take_stack_hash(); \ \ struct name##_args_t a = *args; \ bpf_map_delete_elem(&name##_args, &tid); \ @@ -77,20 +83,22 @@ submit_block; \ } -UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), { return submit_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(malloc, PT_REGS_PARM1(ctx), + { return submit_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0); }) +UPROBE_RET(free, PT_REGS_PARM1(ctx), { return submit_free_event(arg0, stack_hash); }) UPROBE_ARG_RET(calloc, PT_REGS_PARM1(ctx) * PT_REGS_PARM2(ctx), - { return submit_calloc_event(arg0, ret_val); }) + { return submit_calloc_event(arg0, ret_val, stack_hash); }) UPROBE_ARGS_RET(realloc, PT_REGS_PARM2(ctx), PT_REGS_PARM1(ctx), - { return submit_realloc_event(arg1, ret_val, arg0); }) + { return submit_realloc_event(arg1, ret_val, arg0, stack_hash); }) UPROBE_ARG_RET(aligned_alloc, PT_REGS_PARM2(ctx), - { return submit_aligned_alloc_event(arg0, ret_val); }) + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) -UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), { return submit_aligned_alloc_event(arg0, ret_val); }) +UPROBE_ARG_RET(memalign, PT_REGS_PARM2(ctx), + { return submit_aligned_alloc_event(arg0, ret_val, stack_hash); }) /* * posix_memalign(void** memptr, size_t alignment, size_t size) @@ -115,6 +123,8 @@ int uprobe_posix_memalign(struct pt_regs* ctx) { return 0; } + stash_stack_hash(capture_stack(ctx)); + struct posix_memalign_args_t args = {.memptr = PT_REGS_PARM1(ctx), .size = PT_REGS_PARM3(ctx)}; bpf_map_update_elem(&posix_memalign_args, &tid, &args, BPF_ANY); return 0; @@ -127,6 +137,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { if (!args) { return 0; } + __u64 stack_hash = take_stack_hash(); struct posix_memalign_args_t a = *args; bpf_map_delete_elem(&posix_memalign_args, &tid); @@ -140,7 +151,7 @@ int uretprobe_posix_memalign(struct pt_regs* ctx) { return 0; } - return submit_aligned_alloc_event(a.size, addr); + return submit_aligned_alloc_event(a.size, addr, stack_hash); } struct mmap_args { diff --git a/crates/memtrack/src/ebpf/c/attach.h b/crates/memtrack/src/ebpf/c/attach.h index e188c7d5f..90cbe4360 100644 --- a/crates/memtrack/src/ebpf/c/attach.h +++ b/crates/memtrack/src/ebpf/c/attach.h @@ -14,11 +14,6 @@ #define MEMTRACK_PROT_EXEC 0x4 #define MEMTRACK_SIGSTOP 19 -struct inode_key { - __u64 dev; - __u64 ino; -}; - /* (dev, ino) -> 1; populated by userspace after classify/attach */ BPF_HASH_MAP(known_inodes, struct inode_key, __u8, 8192); /* Requests are 24 B and rare; overflow aborts the run via the counter below */ diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index bf0677c93..eedb0bd3b 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -15,6 +15,48 @@ #define EVENT_TYPE_RSS 12 #define EVENT_TYPE_RMAP 13 +/* Largest user-stack copy one definition can carry. The scratch buffer holding + * header plus bytes is a per-CPU map value, capped at PCPU_MIN_UNIT_SIZE + * (32 KiB) by the kernel allocator. */ +#define MEMTRACK_MAX_STACK_COPY (32 * 1024 - 512) + +/* Registers, indexed by the capturing architecture's DWARF register number + * (x86_64: 0=rax .. 7=rsp, 8..15=r8-r15, 16=rip; aarch64: 0..30=x0-x30, + * 31=sp, 32=pc). Slots the architecture does not define stay zero. An offline + * DWARF unwinder needs the callee-saved ones to evaluate CFA rules, not just + * ip/sp/bp. */ +#define MEMTRACK_STACK_REGS 33 + +/* Counter slots in the stack_counters array map. */ +#define MEMTRACK_STACK_COUNTER_COPY_FAILED 0 +#define MEMTRACK_STACK_COUNTER_HASH_MAP_FULL 1 +/* bpf_get_stackid() has several negative outcomes (no user callchain, + * hash-bucket collision, or no free bucket), so this counts only missing ids. */ +#define MEMTRACK_STACK_COUNTER_STACKID_FAILED 2 +#define MEMTRACK_STACK_COUNTER_TRUNCATED 3 +#define MEMTRACK_STACK_COUNTER_RING_FULL 4 +#define MEMTRACK_STACK_COUNTER_PREEMPTED 5 +#define MEMTRACK_STACK_COUNTER_COUNT 6 + +struct stack_regs { + uint64_t reg[MEMTRACK_STACK_REGS]; +}; + +/* Head of a stack record; `copy_len` raw stack bytes read upwards from `sp` + * follow it. */ +struct stack_header { + uint64_t hash; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + int64_t stackid; /* bpf_get_stackid() result; negative means unavailable */ + uint64_t sp; /* user stack pointer the copy starts at */ + uint32_t pid; + uint32_t tid; + uint32_t copy_len; + uint8_t truncated; /* the copy hit the size cap */ + uint8_t _pad[3]; + struct stack_regs regs; +}; + /* Common header shared by all event types */ struct event_header { uint8_t event_type; /* See EVENT_TYPE_* constants above */ @@ -29,20 +71,23 @@ struct event { union { /* Allocation events (malloc, calloc, aligned_alloc) */ struct { - uint64_t addr; /* address returned */ - uint64_t size; /* size requested */ + uint64_t addr; /* address returned */ + uint64_t size; /* size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } alloc; /* Deallocation event (free) */ struct { - uint64_t addr; /* address to free */ + uint64_t addr; /* address to free */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } free; /* Reallocation event - includes both old and new addresses */ struct { - uint64_t old_addr; /* previous address (can be NULL) */ - uint64_t new_addr; /* new address returned */ - uint64_t size; /* new size requested */ + uint64_t old_addr; /* previous address (can be NULL) */ + uint64_t new_addr; /* new address returned */ + uint64_t size; /* new size requested */ + uint64_t stack_hash; /* caller stack identity; 0 = not captured */ } realloc; /* Memory mapping events (mmap, munmap, brk) */ @@ -69,6 +114,13 @@ struct event { } data; }; +/* Identifies a mapped file across both the attach watcher and the mapping + * recorder. `dev` uses the kernel's s_dev encoding: (major << 20) | minor. */ +struct inode_key { + uint64_t dev; + uint64_t ino; +}; + /* Request from the exec-mapping watcher to the userspace attach worker */ struct attach_request { uint32_t pid; @@ -76,4 +128,18 @@ struct attach_request { uint64_t ino; }; +/* One executable file mapping, mirroring PERF_RECORD_MMAP2. The path is not + * here: it is resolved once per inode into a BPF map that userspace joins + * against, since every mapping of the same file shares it. */ +struct mapping_record { + uint64_t dev; + uint64_t ino; + uint64_t file_offset; /* offset of the mapping's first byte in the file */ + uint64_t start; + uint64_t end; + uint64_t timestamp; /* monotonic time in nanoseconds (CLOCK_MONOTONIC) */ + uint32_t pid; + uint32_t _pad; +}; + #endif /* __EVENT_H__ */ diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 5a8d6ff07..7a068c605 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,9 +8,11 @@ #include "allocator.h" #include "attach.h" #include "event.h" +#include "mappings.bpf.h" #include "process_tracking.bpf.h" #include "rmap.bpf.h" #include "rss.bpf.h" +#include "stack_capture.bpf.h" #include "utils/event_helpers.h" #include "utils/folio.h" #include "utils/map_helpers.h" diff --git a/crates/memtrack/src/ebpf/c/mappings.bpf.h b/crates/memtrack/src/ebpf/c/mappings.bpf.h new file mode 100644 index 000000000..71b11b391 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/mappings.bpf.h @@ -0,0 +1,166 @@ +#ifndef __MAPPINGS_BPF_H__ +#define __MAPPINGS_BPF_H__ + +#include "event.h" +#include "utils/folio.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* == Mapping recorder == + * + * Reconstructs what `PERF_RECORD_MMAP2` gives perf: which file a tracked + * process mapped, where, so raw stack addresses can be attributed to modules + * offline. No single hook carries both halves: + * + * security_mmap_file(file, ..) has the file, runs before the VMA exists + * perf_event_mmap(vma) has the addresses, cannot resolve a path + * + * The path therefore lands in a per-inode cache, and the address-bearing hook + * emits inode-keyed records that userspace joins against that cache while this + * BPF object is still loaded. + * + * Path resolution is only reachable from an LSM program: `bpf_d_path()` is + * restricted to sleepable LSM hooks, `BPF_TRACE_ITER` and an fentry allowlist + * holding no mmap path, and the newer `bpf_path_d_path()` kfunc rejects + * non-LSM program types. Both variants are compiled; userspace autoloads the + * one the running kernel supports and neither when the bpf LSM is inactive. */ + +/* VM_EXEC from linux/mm.h, which vmlinux.h does not carry (it is a macro, not a + * type). Only executable mappings are recorded: unwind data and symbols are + * looked up by text address. */ +#define MEMTRACK_VM_EXEC 0x00000004 + +/* d_path() fails with -ENAMETOOLONG rather than truncating, so a short buffer + * loses whole modules. PATH_MAX keeps that from happening. */ +#define MEMTRACK_MAX_PATH 4096 + +struct inode_path { + __u32 len; /* bytes written by d_path, including the NUL */ + char path[MEMTRACK_MAX_PATH]; +}; + +/* Resolved once per mapped file: a run maps hundreds of distinct files, not + * thousands, and every mapping of the same inode shares the path. */ +BPF_HASH_MAP(path_by_inode, struct inode_key, struct inode_path, 2048); + +/* Records are ~64 B and rare (one per executable mapping); the counter below + * reports overflow so the run can fail rather than silently lose a module. */ +BPF_RINGBUF(mappings, 256 * 1024); +BPF_ARRAY_MAP(mapping_dropped, __u64, 1); + +/* An `inode_path` is far larger than the 512 B BPF stack allows, so it is built + * here and copied into the cache from this pointer. */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct inode_path); +} path_scratch SEC(".maps"); + +extern int bpf_path_d_path(const struct path* path, char* buf, __u64 buf__sz) __ksym __weak; + +static __always_inline void bump_mapping_dropped(void) { + __u32 zero = 0; + __u64* drops = bpf_map_lookup_elem(&mapping_dropped, &zero); + if (drops) { + __sync_fetch_and_add(drops, 1); + } +} + +/* The scratch buffer to resolve `file`'s path into, or NULL when this mapping + * needs no resolution (untracked process, or the inode is already cached). + * `key` is filled in for the matching [`commit_mapping_path`]. */ +static __always_inline struct inode_path* mapping_path_slot(struct file* file, + struct inode_key* key) { + if (!file || !is_tracked(current_tgid())) { + return NULL; + } + + key->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); + key->ino = BPF_CORE_READ(file, f_inode, i_ino); + if (bpf_map_lookup_elem(&path_by_inode, key)) { + return NULL; + } + + __u32 zero = 0; + return bpf_map_lookup_elem(&path_scratch, &zero); +} + +/* Publish a resolved path. A failed resolution is not cached, so the next + * mapping of the same inode retries instead of losing the module for the run. */ +static __always_inline void commit_mapping_path(struct inode_key* key, struct inode_path* entry, + int len) { + if (len <= 0) { + return; + } + entry->len = (__u32)len; + bpf_map_update_elem(&path_by_inode, key, entry, BPF_NOEXIST); +} + +/* Kernels >= 6.12: the kfunc is callable from any LSM program. */ +SEC("lsm/mmap_file") +int BPF_PROG(cache_mmap_path_kfunc, struct file* file, unsigned long reqprot, unsigned long prot, + unsigned long flags) { + struct inode_key key = {}; + struct inode_path* entry = mapping_path_slot(file, &key); + if (entry) { + commit_mapping_path(&key, entry, + bpf_path_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); + } + return 0; +} + +/* Kernels 5.11..6.11: `bpf_d_path()` needs a sleepable LSM hook, which + * `mmap_file` has been since 5.11. */ +SEC("lsm.s/mmap_file") +int BPF_PROG(cache_mmap_path_legacy, struct file* file, unsigned long reqprot, unsigned long prot, + unsigned long flags) { + struct inode_key key = {}; + struct inode_path* entry = mapping_path_slot(file, &key); + if (entry) { + commit_mapping_path(&key, entry, bpf_d_path(&file->f_path, entry->path, MEMTRACK_MAX_PATH)); + } + return 0; +} + +/* The same hook perf emits MMAP2 from, so the recorded geometry matches what + * the walltime pipeline already consumes: the file offset is in bytes, not + * pages. */ +SEC("fentry/perf_event_mmap") +int BPF_PROG(record_mmap, struct vm_area_struct* vma) { + if (!vma) { + return 0; + } + + __u32 tgid = current_tgid(); + if (!is_tracked(tgid)) { + return 0; + } + + struct file* file = BPF_CORE_READ(vma, vm_file); + if (!file) { + return 0; + } + if (!(BPF_CORE_READ(vma, vm_flags) & MEMTRACK_VM_EXEC)) { + return 0; + } + + struct mapping_record* rec = bpf_ringbuf_reserve(&mappings, sizeof(*rec), 0); + if (!rec) { + bump_mapping_dropped(); + return 0; + } + + rec->pid = tgid; + rec->dev = BPF_CORE_READ(file, f_inode, i_sb, s_dev); + rec->ino = BPF_CORE_READ(file, f_inode, i_ino); + rec->file_offset = (__u64)BPF_CORE_READ(vma, vm_pgoff) << page_shift; + rec->start = BPF_CORE_READ(vma, vm_start); + rec->end = BPF_CORE_READ(vma, vm_end); + rec->timestamp = bpf_ktime_get_ns(); + bpf_ringbuf_submit(rec, 0); + + return 0; +} + +#endif /* __MAPPINGS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/stack_capture.bpf.h b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h new file mode 100644 index 000000000..28f7fcf88 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/stack_capture.bpf.h @@ -0,0 +1,275 @@ +#ifndef __STACK_CAPTURE_BPF_H__ +#define __STACK_CAPTURE_BPF_H__ + +#include "event.h" +#include "utils/map_helpers.h" +#include "utils/process_tracking.h" + +/* At allocator entry the caller's raw user stack is copied and hashed; the hash + * travels on the allocation event as its stack identity. The first time a hash + * is seen, the copied bytes plus a register snapshot are emitted as a stack + * record so the stack can be DWARF-unwound offline, with an in-kernel + * frame-pointer walk alongside as the fallback. + * + * Hashing raw bytes rather than unwound frames splits one call path into + * several identities whenever locals or arguments in the copied region differ. + * That only costs extra records; the reverse trade (aliasing) would corrupt + * attribution. + */ + +const volatile __u8 capture_stacks_enabled = 0; +const volatile __u32 stack_copy_size = 8192; + +#define STACK_TRACE_MAX_DEPTH 127 +/* Copy granularity: a recovered length is exact only to within one chunk. */ +#define STACK_COPY_CHUNK 512 +#define FNV64_OFFSET 0xcbf29ce484222325ULL +#define FNV64_PRIME 0x00000100000001b3ULL + +/* Frame-pointer walk results, indexed by the id bpf_get_stackid() returns. */ +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(max_entries, 16384); + __type(key, __u32); + __uint(value_size, STACK_TRACE_MAX_DEPTH * sizeof(__u64)); +} stack_traces SEC(".maps"); + +/* Records are bulky but rare (one per distinct hash), so they get their own + * ring rather than inflating the fixed-size `struct event` path. */ +BPF_RINGBUF(stacks, 64 * 1024 * 1024); +BPF_HASH_MAP(seen_stack_hashes, __u64, __u8, 65536); +BPF_HASH_MAP(pending_stack_hash, __u64, __u64, 10000); +BPF_ARRAY_MAP(stack_counters, __u64, MEMTRACK_STACK_COUNTER_COUNT); + +/* The record is built in place here and handed to the ring buffer as one + * contiguous variable-length blob. `words` aliases `bytes` so the hash loop + * reads whole registers without a per-byte shift chain. */ +struct stack_scratch_buf { + struct stack_header header; + union { + __u8 bytes[MEMTRACK_MAX_STACK_COPY]; + __u64 words[MEMTRACK_MAX_STACK_COPY / 8]; + }; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, struct stack_scratch_buf); +} stack_scratch SEC(".maps"); + +/* The value is 64-bit because the BPF backend of older clang cannot select a + * 32-bit atomic compare-and-swap. */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(max_entries, 1); + __type(key, __u32); + __type(value, __u64); +} stack_busy SEC(".maps"); + +static __always_inline void bump_stack_counter(__u32 index) { + __u64* slot = bpf_map_lookup_elem(&stack_counters, &index); + if (slot) { + __sync_fetch_and_add(slot, 1); + } +} + +#if defined(__TARGET_ARCH_x86) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + out->reg[0] = ctx->ax; + out->reg[1] = ctx->dx; + out->reg[2] = ctx->cx; + out->reg[3] = ctx->bx; + out->reg[4] = ctx->si; + out->reg[5] = ctx->di; + out->reg[6] = ctx->bp; + out->reg[7] = ctx->sp; + out->reg[8] = ctx->r8; + out->reg[9] = ctx->r9; + out->reg[10] = ctx->r10; + out->reg[11] = ctx->r11; + out->reg[12] = ctx->r12; + out->reg[13] = ctx->r13; + out->reg[14] = ctx->r14; + out->reg[15] = ctx->r15; + out->reg[16] = ctx->ip; +} +#elif defined(__TARGET_ARCH_arm64) +static __always_inline void fill_stack_regs(struct stack_regs* out, struct pt_regs* ctx) { + struct user_pt_regs* uregs = (struct user_pt_regs*)ctx; +#pragma unroll + for (int i = 0; i < 31; i++) { + out->reg[i] = uregs->regs[i]; + } + out->reg[31] = uregs->sp; + out->reg[32] = uregs->pc; +} +#else +#error "stack capture needs a DWARF register mapping for this architecture" +#endif + +/* Returns the stack identity, or 0 when nothing could be copied. */ +static __always_inline __u64 capture_stack_inner(struct pt_regs* ctx, struct task_ids ids) { + __u32 zero = 0; + struct stack_scratch_buf* scratch = bpf_map_lookup_elem(&stack_scratch, &zero); + if (!scratch) { + return 0; + } + + __u64 sp = PT_REGS_SP(ctx); + __u32 want = stack_copy_size; + if (want > MEMTRACK_MAX_STACK_COPY) { + want = MEMTRACK_MAX_STACK_COPY; + } + want &= ~(__u32)(STACK_COPY_CHUNK - 1); + if (want < STACK_COPY_CHUNK) { + want = STACK_COPY_CHUNK; + } + + /* bpf_probe_read_user() is all-or-nothing and the readable region ends at + * the top of the stack mapping, which is not knowable up front, so the copy + * advances in chunks and stops at the first unreadable one. + * + * Each chunk is hashed as it lands, over a constant iteration count the + * compiler fully unrolls. One loop over the whole copy instead costs the + * verifier a state fork per word and blows the one-million instruction + * budget well below the maximum copy size. */ + __u64 hash = FNV64_OFFSET; + __u32 got = 0; +#pragma clang loop unroll(disable) + for (__u32 off = 0; off + STACK_COPY_CHUNK <= MEMTRACK_MAX_STACK_COPY; + off += STACK_COPY_CHUNK) { + if (off >= want) { + break; + } + if (bpf_probe_read_user(&scratch->bytes[off], STACK_COPY_CHUNK, (void*)(sp + off)) != 0) { + break; + } + + __u32 base = off >> 3; +#pragma unroll + for (__u32 word = 0; word < STACK_COPY_CHUNK / 8; word++) { + hash = (hash ^ scratch->words[base + word]) * FNV64_PRIME; + } + got = off + STACK_COPY_CHUNK; + } + + if (got == 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_COPY_FAILED); + return 0; + } + + __u8 truncated = got >= want; + if (truncated) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_TRUNCATED); + } + + /* Fold in the length so a truncated prefix of a deep stack cannot collide + * with a full copy of a shallower one, and keep 0 reserved as the "no + * stack" marker on allocation events. */ + hash = (hash ^ got) * FNV64_PRIME; + if (hash == 0) { + hash = FNV64_OFFSET; + } + + __u8 marker = 1; + long gate_result = bpf_map_update_elem(&seen_stack_hashes, &hash, &marker, BPF_NOEXIST); + if (gate_result == -17) { /* -EEXIST: already emitted */ + return hash; + } + if (gate_result != 0) { + /* A full gate cannot retain this identity, so emit it on every + * occurrence rather than make the allocation hash unresolvable. */ + bump_stack_counter(MEMTRACK_STACK_COUNTER_HASH_MAP_FULL); + } + + __s64 stackid = bpf_get_stackid(ctx, &stack_traces, BPF_F_USER_STACK); + if (stackid < 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_STACKID_FAILED); + } + + scratch->header.hash = hash; + scratch->header.timestamp = bpf_ktime_get_ns(); + scratch->header.stackid = stackid; + scratch->header.sp = sp; + scratch->header.pid = ids.tgid; + scratch->header.tid = ids.tid; + scratch->header.copy_len = got; + scratch->header.truncated = truncated; + scratch->header._pad[0] = 0; + scratch->header._pad[1] = 0; + scratch->header._pad[2] = 0; + fill_stack_regs(&scratch->header.regs, ctx); + + if (bpf_ringbuf_output(&stacks, scratch, sizeof(struct stack_header) + got, 0) != 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_RING_FULL); + bpf_map_delete_elem(&seen_stack_hashes, &hash); + } + + return hash; +} + +/* Copy and hash the caller's stack, emitting a record on first sight of the + * resulting identity. Returns 0 when capture is off or nothing was copied. */ +static __always_inline __u64 capture_stack(struct pt_regs* ctx) { + if (!capture_stacks_enabled || !is_enabled()) { + return 0; + } + + struct task_ids ids = current_task_ids(); + if (!is_tracked(ids.tgid)) { + return 0; + } + + __u32 zero = 0; + __u64* busy = bpf_map_lookup_elem(&stack_busy, &zero); + if (!busy) { + return 0; + } + /* uprobe_multi runs programs without the bpf_prog_active recursion guard, + * so a task preempting this one on the same CPU could corrupt the scratch. */ + if (__sync_val_compare_and_swap(busy, 0, 1) != 0) { + bump_stack_counter(MEMTRACK_STACK_COUNTER_PREEMPTED); + return 0; + } + + __u64 hash = capture_stack_inner(ctx, ids); + + /* A plain store, not an atomic release: the only contender is a task that + * preempted this one on this same CPU, and the context switch between them + * already orders the write. The BPF backend cannot select a release store. */ + *busy = 0; + return hash; +} + +/* Hand an identity to the matching uretprobe. */ +static __always_inline void stash_stack_hash(__u64 hash) { + if (hash == 0) { + return; + } + + __u64 tid = current_tid(); + bpf_map_update_elem(&pending_stack_hash, &tid, &hash, BPF_ANY); +} + +/* The identity stashed by the matching entry probe, or 0 when capture is off or + * the entry probe bailed out. The slot is per-thread but shared by allocators, + * so every return path must clear it. */ +static __always_inline __u64 take_stack_hash(void) { + if (!capture_stacks_enabled) { + return 0; + } + + __u64 tid = current_tid(); + __u64* hash = bpf_map_lookup_elem(&pending_stack_hash, &tid); + if (!hash) { + return 0; + } + + __u64 value = *hash; + bpf_map_delete_elem(&pending_stack_hash, &tid); + return value; +} + +#endif /* __STACK_CAPTURE_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index ca5969a9a..ca53593d2 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -2,6 +2,7 @@ #define __EVENT_HELPERS_H__ #include "../event.h" +#include "../stack_capture.bpf.h" #include "map_helpers.h" #include "process_tracking.h" @@ -88,36 +89,44 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(owner.tgid, evt_type, fill_data); \ } -static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { +static __always_inline int submit_calloc_event(__u64 size, __u64 addr, __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; + e->data.alloc.stack_hash = stack_hash; }); } -static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); +static __always_inline int submit_free_event(__u64 addr, __u64 stack_hash) { + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { + e->data.free.addr = addr; + e->data.free.stack_hash = stack_hash; + }); } -static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { +static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size, + __u64 stack_hash) { SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; + e->data.realloc.stack_hash = stack_hash; }); } diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 4ed422a58..795973572 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -34,13 +34,20 @@ pub fn parse_event(data: &[u8]) -> Option { event.data.alloc.addr, MemtrackEventKind::Malloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, + }, + ), + EVENT_TYPE_FREE => ( + event.data.free.addr, + MemtrackEventKind::Free { + stack_hash: event.data.free.stack_hash, }, ), - EVENT_TYPE_FREE => (event.data.free.addr, MemtrackEventKind::Free), EVENT_TYPE_CALLOC => ( event.data.alloc.addr, MemtrackEventKind::Calloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_REALLOC => ( @@ -48,12 +55,14 @@ pub fn parse_event(data: &[u8]) -> Option { MemtrackEventKind::Realloc { old_addr: Some(event.data.realloc.old_addr), size: event.data.realloc.size, + stack_hash: event.data.realloc.stack_hash, }, ), EVENT_TYPE_ALIGNED_ALLOC => ( event.data.alloc.addr, MemtrackEventKind::AlignedAlloc { size: event.data.alloc.size, + stack_hash: event.data.alloc.stack_hash, }, ), EVENT_TYPE_MMAP => ( @@ -157,6 +166,7 @@ mod tests { event.data.realloc.old_addr = 0x1000; event.data.realloc.new_addr = 0x2000; event.data.realloc.size = 256; + event.data.realloc.stack_hash = 0xbeef; let bytes = event_bytes(&event); @@ -168,9 +178,14 @@ mod tests { assert_eq!(parsed.addr, 0x2000); match parsed.kind { - MemtrackEventKind::Realloc { old_addr, size } => { + MemtrackEventKind::Realloc { + old_addr, + size, + stack_hash, + } => { assert_eq!(old_addr, Some(0x1000)); assert_eq!(size, 256); + assert_eq!(stack_hash, 0xbeef); } _ => panic!("Expected Realloc event kind"), } @@ -186,6 +201,7 @@ mod tests { event.header.tid = 2000; event.data.alloc.addr = 0x1000; event.data.alloc.size = 128; + event.data.alloc.stack_hash = 0x1234; let bytes = event_bytes(&event); @@ -197,8 +213,9 @@ mod tests { assert_eq!(parsed.addr, 0x1000); match parsed.kind { - MemtrackEventKind::Malloc { size } => { + MemtrackEventKind::Malloc { size, stack_hash } => { assert_eq!(size, 128); + assert_eq!(stack_hash, 0x1234); } _ => panic!("Expected Malloc event kind"), } diff --git a/crates/memtrack/src/ebpf/mappings/mod.rs b/crates/memtrack/src/ebpf/mappings/mod.rs new file mode 100644 index 000000000..581de916b --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/mod.rs @@ -0,0 +1,7 @@ +mod records; +mod resolve; +mod support; + +pub(crate) use records::MappingRecord; +pub(crate) use resolve::resolve_mappings; +pub use support::MappingSupport; diff --git a/crates/memtrack/src/ebpf/mappings/records.rs b/crates/memtrack/src/ebpf/mappings/records.rs new file mode 100644 index 000000000..0d9609ffc --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/records.rs @@ -0,0 +1,84 @@ +use crate::ebpf::events::bindings::mapping_record; + +/// One executable file mapping as the BPF recorder saw it. The path is resolved +/// separately, per inode. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MappingRecord { + pub pid: u32, + pub dev: u64, + pub ino: u64, + pub file_offset: u64, + pub start: u64, + pub end: u64, + pub timestamp: u64, +} + +impl MappingRecord { + /// Decode one record from raw ring buffer bytes. + pub fn parse(data: &[u8]) -> Option { + if data.len() < std::mem::size_of::() { + return None; + } + + // SAFETY: the length is checked above, and the layout is the + // bindgen-generated C ABI struct. + let record: mapping_record = unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) }; + Some(Self { + pid: record.pid, + dev: record.dev, + ino: record.ino, + file_offset: record.file_offset, + start: record.start, + end: record.end, + timestamp: record.timestamp, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn encode(record: mapping_record) -> Vec { + // SAFETY: reading a plain-data struct as bytes. + unsafe { + std::slice::from_raw_parts( + (&record as *const mapping_record).cast::(), + std::mem::size_of::(), + ) + } + .to_vec() + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let bytes = encode(mapping_record { + dev: 0x1_0002, + ino: 4242, + file_offset: 0x2000, + start: 0x5555_5555_0000, + end: 0x5555_5556_0000, + timestamp: 987_654_321, + pid: 7, + _pad: 0, + }); + + assert_eq!( + MappingRecord::parse(&bytes), + Some(MappingRecord { + pid: 7, + dev: 0x1_0002, + ino: 4242, + file_offset: 0x2000, + start: 0x5555_5555_0000, + end: 0x5555_5556_0000, + timestamp: 987_654_321, + }) + ); + } + + #[test] + fn truncated_buffer_returns_none() { + assert!(MappingRecord::parse(&[0u8; 8]).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/mappings/resolve.rs b/crates/memtrack/src/ebpf/mappings/resolve.rs new file mode 100644 index 000000000..bc2e24ad6 --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/resolve.rs @@ -0,0 +1,77 @@ +use super::MappingRecord; +use crate::prelude::*; +use runner_shared::artifacts::ProcessMapping; +use std::collections::HashMap; + +/// Join recorded mappings with the per-inode paths resolved in the kernel. +/// +/// A record whose inode has no path is dropped: it was mapped by a process the +/// LSM hook never saw resolve, and without a path there is nothing to read +/// unwind data or symbols from. +pub(crate) fn resolve_mappings( + records: Vec, + paths: &HashMap<(u64, u64), String>, +) -> Vec { + let mut unresolved = 0; + let mappings = records + .into_iter() + .filter_map(|record| { + let Some(path) = paths.get(&(record.dev, record.ino)) else { + unresolved += 1; + return None; + }; + + Some(ProcessMapping { + pid: record.pid as i32, + path: path.clone(), + dev: record.dev, + ino: record.ino, + file_offset: record.file_offset, + avma_range: record.start..record.end, + timestamp: record.timestamp, + }) + }) + .collect(); + + if unresolved > 0 { + debug!("{unresolved} mapping records had no resolved path and were dropped"); + } + mappings +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(dev: u64, ino: u64) -> MappingRecord { + MappingRecord { + pid: 5, + dev, + ino, + file_offset: 0x1000, + start: 0x4000, + end: 0x8000, + timestamp: 42, + } + } + + #[test] + fn resolves_records_against_the_path_cache() { + let paths = HashMap::from([((1, 2), "/lib/libc.so.6".to_string())]); + + let mappings = resolve_mappings(vec![record(1, 2)], &paths); + + assert_eq!(mappings.len(), 1); + assert_eq!(mappings[0].path, "/lib/libc.so.6"); + assert_eq!(mappings[0].avma_range, 0x4000..0x8000); + assert_eq!(mappings[0].file_offset, 0x1000); + assert_eq!(mappings[0].pid, 5); + } + + /// A module we cannot name is a module we cannot read, so it must not reach + /// the artifact as an empty path. + #[test] + fn drops_records_without_a_resolved_path() { + assert!(resolve_mappings(vec![record(9, 9)], &HashMap::new()).is_empty()); + } +} diff --git a/crates/memtrack/src/ebpf/mappings/support.rs b/crates/memtrack/src/ebpf/mappings/support.rs new file mode 100644 index 000000000..9e2f3a190 --- /dev/null +++ b/crates/memtrack/src/ebpf/mappings/support.rs @@ -0,0 +1,103 @@ +use crate::kernel::KernelVersion; +use crate::prelude::*; + +/// How the running kernel can resolve a mapped file's path inside BPF. +/// +/// Only a BPF LSM program can do it at all: `bpf_d_path()` is restricted to +/// `BPF_TRACE_ITER` programs, sleepable LSM hooks and a fixed fentry allowlist +/// that contains no mmap path (`bpf_d_path_allowed()` in +/// `kernel/trace/bpf_trace.c`), and the `bpf_path_d_path()` kfunc that replaces +/// it rejects every program type but LSM (`bpf_fs_kfuncs_filter()` in +/// `fs/bpf_fs_kfuncs.c`). +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum MappingSupport { + /// Paths cannot be resolved, so allocation stacks could not be attributed to + /// modules and are not worth capturing. + Unsupported, + /// Sleepable LSM hook calling `bpf_d_path()` (kernel >= 5.11). + Legacy, + /// LSM hook calling the `bpf_path_d_path()` kfunc (kernel >= 6.12). + Kfunc, +} + +impl MappingSupport { + /// What the running kernel and its boot configuration provide. + /// + /// The kernel release is only half the gate: `bpf` must also be in the + /// active LSM list, which is fixed at boot by `CONFIG_LSM`/`lsm=` and cannot + /// be inferred from the version. + pub fn detect() -> Self { + if !bpf_lsm_active() { + info!( + "The bpf LSM is not active (see /sys/kernel/security/lsm), so mapped module paths \ + cannot be resolved" + ); + return Self::Unsupported; + } + + let version = match KernelVersion::current() { + Ok(version) => version, + Err(e) => { + warn!("Failed to read the kernel version, no mapping records: {e:#}"); + return Self::Unsupported; + } + }; + + let support = Self::for_version(version); + match support { + Self::Unsupported => { + info!("Kernel {version} cannot resolve paths from an LSM program (needs >= 5.11)") + } + Self::Legacy => { + debug!("Kernel {version} predates the bpf_path_d_path kfunc, using bpf_d_path") + } + Self::Kfunc => {} + } + support + } + + fn for_version(version: KernelVersion) -> Self { + if version < KernelVersion::new(5, 11) { + return Self::Unsupported; + } + if version < KernelVersion::new(6, 12) { + return Self::Legacy; + } + Self::Kfunc + } +} + +/// Whether `bpf` is one of the LSMs the running kernel initialized. An +/// unreadable file means securityfs is not mounted, in which case no LSM program +/// will attach either. +fn bpf_lsm_active() -> bool { + const PATH: &str = "/sys/kernel/security/lsm"; + + let Ok(active) = std::fs::read_to_string(PATH) else { + debug!("Could not read {PATH} to check whether the bpf LSM is active"); + return false; + }; + active.trim().split(',').any(|lsm| lsm == "bpf") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// `bpf_lsm_mmap_file` has been in the sleepable hook set since 5.11, and + /// 6.12 is the first release carrying `bpf_path_d_path`. + #[test] + fn maps_releases_to_support_levels() { + for (major, minor, expected) in [ + (5, 4, MappingSupport::Unsupported), + (5, 10, MappingSupport::Unsupported), + (5, 11, MappingSupport::Legacy), + (6, 11, MappingSupport::Legacy), + (6, 12, MappingSupport::Kfunc), + (7, 1, MappingSupport::Kfunc), + ] { + let version = KernelVersion::new(major, minor); + assert_eq!(MappingSupport::for_version(version), expected, "{version}"); + } + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/maps.rs b/crates/memtrack/src/ebpf/memtrack/maps.rs index c7376d463..5a4bf3558 100644 --- a/crates/memtrack/src/ebpf/memtrack/maps.rs +++ b/crates/memtrack/src/ebpf/memtrack/maps.rs @@ -1,6 +1,8 @@ use super::MemtrackBpf; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::prelude::*; use libbpf_rs::MapCore; +use std::collections::HashMap; impl MemtrackBpf { pub fn add_tracked_pid(&mut self, pid: i32) -> Result<()> { @@ -61,6 +63,39 @@ impl MemtrackBpf { ) } + /// Number of mapping records dropped because their ring buffer was full. + /// A non-zero value means a module may be missing from the trace. + pub fn mapping_dropped_count(&self) -> Result { + read_counter( + with_skel!(self, skel => &skel.maps.mapping_dropped), + "mapping_dropped", + ) + } + + /// The paths the kernel resolved for every mapped file, keyed by + /// `(dev, ino)`. Only readable while the BPF object is alive. + pub fn mapped_paths(&self) -> Result> { + let map = with_skel!(self, skel => &skel.maps.path_by_inode); + + let mut paths = HashMap::new(); + for key in map.keys() { + let Some(value) = map + .lookup(&key, libbpf_rs::MapFlags::ANY) + .context("Failed to read a resolved mapping path")? + else { + continue; + }; + + let Some((dev, ino)) = inode_key(&key) else { + continue; + }; + if let Some(path) = inode_path(&value) { + paths.insert((dev, ino), path); + } + } + Ok(paths) + } + pub fn dropped_events_count(&self) -> Result { read_counter( with_skel!(self, skel => &skel.maps.dropped_events), @@ -68,6 +103,10 @@ impl MemtrackBpf { ) } + pub fn stack_capture_stats(&self) -> Result { + StackCaptureStats::read(with_skel!(self, skel => &skel.maps.stack_counters)) + } + pub fn ownership_maps(&self) -> Result { let owner_by_mm = entries(with_skel!(self, skel => &skel.maps.owner_by_mm))?; let mm_by_pid = entries(with_skel!(self, skel => &skel.maps.mm_by_pid))?; @@ -113,6 +152,24 @@ fn le(bytes: &[u8]) -> u64 { .fold(0, |acc, &b| acc << 8 | u64::from(b)) } +/// Split a `struct inode_key { __u64 dev; __u64 ino; }` map key. +fn inode_key(key: &[u8]) -> Option<(u64, u64)> { + if key.len() < 16 { + return None; + } + Some((le(&key[..8]), le(&key[8..16]))) +} + +/// Read a `struct inode_path { __u32 len; char path[]; }` map value. The kernel +/// wrote `len` bytes including the NUL terminator. +fn inode_path(value: &[u8]) -> Option { + const PATH_OFFSET: usize = 4; + + let len = u32::from_le_bytes(value.get(..PATH_OFFSET)?.try_into().ok()?) as usize; + let path = value.get(PATH_OFFSET..PATH_OFFSET + len.saturating_sub(1))?; + Some(String::from_utf8_lossy(path).into_owned()) +} + /// Read slot 0 of a single-entry `__u64` array map. fn read_counter(map: &impl MapCore, name: &str) -> Result { let key = 0u32; diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 8586872e6..72171685b 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::mem::MaybeUninit; use std::path::Path; +use crate::ebpf::mappings::MappingSupport; use crate::ebpf::poller::RingBufferPoller; mod token { @@ -20,11 +21,14 @@ mod macros; mod allocator; mod maps; mod rmap; +mod teardown; mod tracking; pub use maps::OwnershipMaps; pub use rmap::RmapSupport; +use teardown::FdHolder; + use crate::bpf_token::has_delegated_bpf_token; /// Which attach mechanism a loaded skeleton uses for its uprobes. See @@ -119,23 +123,36 @@ pub struct MemtrackBpf { pub(super) skel: Skel, pub(super) probes: Vec, rmap: RmapSupport, + pub(super) mappings: MappingSupport, } impl MemtrackBpf { /// Load the skeleton, picking the variant a BPF token is available for. - pub fn new_with_rmap(track_rmap: bool) -> Result { + pub fn new_with_rmap( + track_rmap: bool, + stack_copy_size: Option, + mappings: MappingSupport, + ) -> Result { let variant = if has_delegated_bpf_token() { BpfVariant::Token } else { BpfVariant::Legacy }; - Self::with_variant(variant, track_rmap) + Self::with_variant(variant, track_rmap, stack_copy_size, mappings) } /// Load a specific variant rather than the one [`Self::new_with_rmap`] /// would detect. Either attaches given host privileges; the token only /// matters when `bpf()` is called from an unprivileged user namespace. - pub fn with_variant(variant: BpfVariant, track_rmap: bool) -> Result { + /// + /// `stack_copy_size` turns on allocation stack capture, and `mappings` + /// selects the path-resolving LSM program the running kernel supports. + pub fn with_variant( + variant: BpfVariant, + track_rmap: bool, + stack_copy_size: Option, + mappings: MappingSupport, + ) -> Result { let page_shift = page_shift()?; let rmap = if track_rmap { RmapSupport::detect() @@ -164,6 +181,19 @@ impl MemtrackBpf { rodata.target_pidns_dev = dev; rodata.target_pidns_ino = ino; } + if let Some(copy_size) = stack_copy_size { + rodata.capture_stacks_enabled = 1; + rodata.stack_copy_size = copy_size; + } + } + + // Avoid reserving the stack maps when capture is disabled. A + // ring buffer's size must stay a power-of-two page count. + if stack_copy_size.is_none() { + open_skel.maps.stacks.set_max_entries(4096)?; + open_skel.maps.stack_traces.set_max_entries(1)?; + open_skel.maps.seen_stack_hashes.set_max_entries(1)?; + open_skel.maps.pending_stack_hash.set_max_entries(1)?; } // Autoload is decided before load(), so fentries whose targets @@ -188,6 +218,26 @@ impl MemtrackBpf { RmapSupport::CoreAndPud => {} } + // The kfunc variant fails to load on kernels without + // `bpf_path_d_path`, and neither LSM program can attach when the + // bpf LSM is inactive; without a path there is nothing to + // resolve records against, so the recorder goes too. + match mappings { + MappingSupport::Unsupported => { + open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); + open_skel.progs.cache_mmap_path_legacy.set_autoload(false); + open_skel.progs.record_mmap.set_autoload(false); + open_skel.maps.mappings.set_max_entries(4096)?; + open_skel.maps.path_by_inode.set_max_entries(1)?; + } + MappingSupport::Legacy => { + open_skel.progs.cache_mmap_path_kfunc.set_autoload(false); + } + MappingSupport::Kfunc => { + open_skel.progs.cache_mmap_path_legacy.set_autoload(false); + } + } + $skel(Box::new( open_skel .load() @@ -209,6 +259,7 @@ impl MemtrackBpf { skel, probes: Vec::new(), rmap, + mappings, }) } @@ -227,6 +278,38 @@ impl MemtrackBpf { )) } + /// Poll the stack-record ring buffer into `tx`. + pub(crate) fn poll_stacks( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + use crate::ebpf::stacks::events; + use runner_shared::artifacts::MemtrackEventKind; + + // The poller outlives this borrow of the skeleton, so the chain lookup + // needs an owned handle rather than a reference to the skeleton map. + let stack_traces = with_skel!(self, skel => { + libbpf_rs::MapHandle::try_from(&skel.maps.stack_traces) + .context("Failed to create handle for stack_traces map")? + }); + + let parse = move |data: &[u8]| { + let (mut event, stackid) = events::parse_stack(data)?; + if let MemtrackEventKind::Stack { record } = &mut event.kind { + record.fp_chain = events::fp_chain(&stack_traces, stackid); + } + Some(event) + }; + + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.stacks, + parse, + tx, + poll_interval_ms, + )) + } + /// Poll the exec-mapping request ring buffer into `tx`. Same contract as /// [`Self::poll_events_with_channel`]. pub(crate) fn poll_attach_with_channel( @@ -242,33 +325,62 @@ impl MemtrackBpf { )) } + /// Poll the mapping-record ring buffer into `tx`. Same contract as + /// [`Self::poll_events_with_channel`]. + pub(crate) fn poll_mappings_with_channel( + &self, + poll_interval_ms: u64, + tx: std::sync::mpsc::Sender, + ) -> Result { + with_skel!(self, skel => RingBufferPoller::new( + &skel.maps.mappings, + crate::ebpf::mappings::MappingRecord::parse, + tx, + poll_interval_ms, + )) + } + + /// Whether the mapping recorder is loaded, i.e. whether its ring buffer is + /// worth polling. + pub fn records_mappings(&self) -> bool { + self.mappings != MappingSupport::Unsupported + } + /// Number of currently-attached probes/links. pub fn probe_count(&self) -> usize { self.probes.len() } - /// Detach all BPF links in parallel. Closing a uprobe link blocks on two - /// RCU grace periods in the kernel, but concurrent waiters share grace - /// periods, so closing from many threads scales near-linearly. + /// Detach all BPF links without waiting for the kernel to complete it. + /// + /// Each release waits for a tasks-trace RCU grace period, and that wait is + /// unbounded: a kernel that has stopped completing them (a BPF program that + /// oopsed leaves a reader that never exits) blocks the caller forever, with + /// no way out even by exiting, since exiting closes the same descriptors. A + /// holder process takes them over, so the releases happen off the critical + /// path and the run always finishes with the trace it has. + /// + /// The probes stay installed until the holder is done, so a process starting + /// right after this returns can still trap into them. pub fn detach_probes(&mut self) { - const DETACH_THREADS: usize = 32; - - let mut probes = std::mem::take(&mut self.probes); + let probes = std::mem::take(&mut self.probes); if probes.is_empty() { return; } - debug!("Detaching {} BPF links", probes.len()); - let start = std::time::Instant::now(); - let chunk_size = probes.len().div_ceil(DETACH_THREADS); - std::thread::scope(|scope| { - while !probes.is_empty() { - let split_at = probes.len().saturating_sub(chunk_size); - let chunk = probes.split_off(split_at); - scope.spawn(move || drop(chunk)); + let count = probes.len(); + let holder = FdHolder::fork() + .inspect_err(|e| warn!("Detaching without a descriptor holder: {e:#}")) + .ok(); + drop(probes); + + match holder { + Some(holder) => { + debug!("Releasing {count} BPF links in holder {}", holder.pid()); + holder.release(); } - }); - debug!("Detached BPF links in {:?}", start.elapsed()); + None => debug!("Detached {count} BPF links"), + } } } diff --git a/crates/memtrack/src/ebpf/memtrack/teardown.rs b/crates/memtrack/src/ebpf/memtrack/teardown.rs new file mode 100644 index 000000000..2214ba97a --- /dev/null +++ b/crates/memtrack/src/ebpf/memtrack/teardown.rs @@ -0,0 +1,117 @@ +use crate::prelude::*; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd}; + +/// A forked process that keeps a second reference to every descriptor this +/// process holds, so releasing them here costs nothing. +/// +/// Releasing a classic uprobe link runs `perf_event_detach_bpf_prog()`, which +/// waits for a tasks-trace RCU grace period. That wait is unbounded: a kernel +/// whose grace periods have stalled never completes it, and the process cannot +/// escape it by exiting either, since exiting closes the very same descriptors. +/// A descriptor closed while another process still holds it only drops a +/// refcount, so with a holder alive the release — and the wait — happens in the +/// holder instead, off the critical path of whoever forked it. +pub(super) struct FdHolder { + pid: libc::pid_t, + /// Closing this releases the holder from its wait. + release: OwnedFd, +} + +impl FdHolder { + /// Fork a holder for the descriptors currently open. Descriptors opened + /// afterwards are not covered. + pub(super) fn fork() -> Result { + let (read_end, write_end) = pipe()?; + + // SAFETY: the child touches nothing but the pipe and _exit(), so it + // cannot deadlock on a lock a thread of the parent held across the fork. + let pid = unsafe { libc::fork() }; + ensure!( + pid >= 0, + "fork() failed: {}", + std::io::Error::last_os_error() + ); + + if pid == 0 { + unsafe { hold_until_released(read_end.as_raw_fd(), write_end.as_raw_fd()) }; + } + + Ok(Self { + pid, + release: write_end, + }) + } + + /// Process id of the holder, for logging. + pub(super) fn pid(&self) -> libc::pid_t { + self.pid + } + + /// Let the holder release the descriptors. It is not waited for: it finishes + /// whenever the kernel lets it and is reaped by init. + pub(super) fn release(self) { + drop(self.release); + } +} + +fn pipe() -> Result<(OwnedFd, OwnedFd)> { + let mut fds = [0 as libc::c_int; 2]; + // SAFETY: pipe() writes two descriptors into `fds`. + let rc = unsafe { libc::pipe(fds.as_mut_ptr()) }; + ensure!( + rc == 0, + "pipe() failed: {}", + std::io::Error::last_os_error() + ); + // SAFETY: both descriptors are fresh and owned by this process. + unsafe { Ok((OwnedFd::from_raw_fd(fds[0]), OwnedFd::from_raw_fd(fds[1]))) } +} + +/// Block until the write end is closed, then exit, which closes every inherited +/// descriptor and performs whatever release work the last close of each entails. +/// +/// # Safety +/// +/// Only for the child of a `fork()`: it never returns, and calls nothing that is +/// unsafe to call between `fork()` and `_exit()` in a multi-threaded process. +unsafe fn hold_until_released(read_end: libc::c_int, write_end: libc::c_int) -> ! { + unsafe { + libc::close(write_end); + + let mut byte = 0u8; + loop { + let rc = libc::read(read_end, (&raw mut byte).cast(), 1); + if rc >= 0 || *libc::__errno_location() != libc::EINTR { + break; + } + } + + libc::_exit(0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The holder must outlive its parent's closes and exit only once released: + /// exiting early would make the parent's own close the last one and put the + /// unbounded wait back on the caller. + #[test] + fn holder_exits_only_once_released() { + let holder = FdHolder::fork().unwrap(); + let pid = holder.pid(); + + std::thread::sleep(std::time::Duration::from_millis(50)); + let mut status = 0; + // SAFETY: waitpid() writes only into `status`. + let reaped = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + assert_eq!(reaped, 0, "holder exited before being released"); + + holder.release(); + // SAFETY: waitpid() writes only into `status`. It holds no descriptor + // whose release can block, so it exits as soon as it is released. + let reaped = unsafe { libc::waitpid(pid, &mut status, 0) }; + assert_eq!(reaped, pid, "holder did not exit after being released"); + } +} diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 065a00315..9f526b1cb 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -1,4 +1,5 @@ use super::{MemtrackBpf, RmapSupport}; +use crate::ebpf::mappings::MappingSupport; use crate::prelude::*; use paste::paste; @@ -54,4 +55,26 @@ impl MemtrackBpf { self.probes.push(link); Ok(()) } + + /// Attach the mapping recorder: the LSM hook caching resolved paths and the + /// `perf_event_mmap` fentry emitting the address records. Only the LSM + /// variant the running kernel supports was loaded. + pub fn attach_mapping_recorder(&mut self) -> Result<()> { + let link = match self.mappings { + MappingSupport::Unsupported => return Ok(()), + MappingSupport::Legacy => { + with_skel!(mut self, skel => skel.progs.cache_mmap_path_legacy.attach()) + } + MappingSupport::Kfunc => { + with_skel!(mut self, skel => skel.progs.cache_mmap_path_kfunc.attach()) + } + } + .context("Failed to attach the mmap path resolver")?; + self.probes.push(link); + + let link = with_skel!(mut self, skel => skel.progs.record_mmap.attach()) + .context("Failed to attach the mapping recorder")?; + self.probes.push(link); + Ok(()) + } } diff --git a/crates/memtrack/src/ebpf/mod.rs b/crates/memtrack/src/ebpf/mod.rs index ad556519b..28d412b74 100644 --- a/crates/memtrack/src/ebpf/mod.rs +++ b/crates/memtrack/src/ebpf/mod.rs @@ -1,12 +1,17 @@ mod attach_worker; mod events; +pub(crate) mod mappings; mod memtrack; pub(crate) mod poller; mod proc_fs; mod spawn; +mod stacks; mod tracker; +pub use mappings::MappingSupport; pub use memtrack::{ BpfVariant, MemtrackBpf, OwnershipMaps, ResolvedSymbols, RmapSupport, resolve_symbol_offsets, }; +pub use stacks::config::{DEFAULT_STACK_COPY_SIZE, clamp_copy_size}; +pub use stacks::counters::StackCaptureStats; pub use tracker::Tracker; diff --git a/crates/memtrack/src/ebpf/stacks/config.rs b/crates/memtrack/src/ebpf/stacks/config.rs new file mode 100644 index 000000000..dd99b8c70 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/config.rs @@ -0,0 +1,54 @@ +use crate::ebpf::events::bindings::MEMTRACK_MAX_STACK_COPY; +use crate::prelude::*; + +pub const DEFAULT_STACK_COPY_SIZE: u32 = 8192; + +/// The per-allocation stack copy budget, or `None` when capture was explicitly +/// disabled with `CODSPEED_MEMTRACK_CAPTURE_STACKS=0`. Capture is on by default. +pub fn stack_copy_size_from_env() -> Option { + if std::env::var("CODSPEED_MEMTRACK_CAPTURE_STACKS").as_deref() == Ok("0") { + return None; + } + + let copy_size = match std::env::var("CODSPEED_MEMTRACK_STACK_COPY_SIZE") { + Ok(value) => match value.parse::() { + Ok(size) => size, + Err(error) => { + warn!( + "Invalid CODSPEED_MEMTRACK_STACK_COPY_SIZE {value:?}: {error}; using default" + ); + DEFAULT_STACK_COPY_SIZE + } + }, + Err(_) => DEFAULT_STACK_COPY_SIZE, + }; + + Some(clamp_copy_size(copy_size)) +} + +/// The kernel copies whole chunks, so a budget that is not a multiple of one +/// would hash bytes it never emits. +pub fn clamp_copy_size(copy_size: u32) -> u32 { + const CHUNK: u32 = 512; + (copy_size / CHUNK * CHUNK).clamp(CHUNK, MEMTRACK_MAX_STACK_COPY) +} + +#[cfg(test)] +mod tests { + use super::clamp_copy_size; + + #[test] + fn rounds_down_to_a_whole_chunk() { + assert_eq!(clamp_copy_size(8_700), 8_192); + } + + #[test] + fn clamps_to_low_bound() { + assert_eq!(clamp_copy_size(63), 512); + } + + #[test] + fn clamps_to_high_bound() { + assert_eq!(clamp_copy_size(u32::MAX), 32_256); + } +} diff --git a/crates/memtrack/src/ebpf/stacks/counters.rs b/crates/memtrack/src/ebpf/stacks/counters.rs new file mode 100644 index 000000000..6e6d76b10 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/counters.rs @@ -0,0 +1,38 @@ +use crate::ebpf::events::bindings::*; +use crate::prelude::*; + +#[derive(Debug, Clone, Copy, Default, serde::Serialize)] +pub struct StackCaptureStats { + pub copy_failed: u64, + pub hash_map_full: u64, + pub stackid_failed: u64, + pub truncated: u64, + pub ring_full: u64, + /// Captures skipped because another BPF program used the per-CPU scratch. + pub preempted: u64, +} + +impl StackCaptureStats { + pub fn read(map: &impl libbpf_rs::MapCore) -> Result { + Ok(Self { + copy_failed: slot(map, MEMTRACK_STACK_COUNTER_COPY_FAILED)?, + hash_map_full: slot(map, MEMTRACK_STACK_COUNTER_HASH_MAP_FULL)?, + stackid_failed: slot(map, MEMTRACK_STACK_COUNTER_STACKID_FAILED)?, + truncated: slot(map, MEMTRACK_STACK_COUNTER_TRUNCATED)?, + ring_full: slot(map, MEMTRACK_STACK_COUNTER_RING_FULL)?, + preempted: slot(map, MEMTRACK_STACK_COUNTER_PREEMPTED)?, + }) + } +} + +fn slot(map: &impl libbpf_rs::MapCore, index: u32) -> Result { + let value = map + .lookup(&index.to_ne_bytes(), libbpf_rs::MapFlags::ANY) + .with_context(|| format!("failed to read stack counter {index}"))? + .ok_or_else(|| anyhow!("stack counter slot {index} missing"))?; + let bytes: [u8; 8] = value + .as_slice() + .try_into() + .map_err(|_| anyhow!("stack counter {index} has unexpected size"))?; + Ok(u64::from_ne_bytes(bytes)) +} diff --git a/crates/memtrack/src/ebpf/stacks/events.rs b/crates/memtrack/src/ebpf/stacks/events.rs new file mode 100644 index 000000000..5d1b6fbf5 --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/events.rs @@ -0,0 +1,143 @@ +use crate::ebpf::events::bindings::stack_header; +use crate::prelude::*; +use libbpf_rs::MapCore; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind, StackRecord}; + +/// Decode one stack record from the ring buffer, returning it alongside the +/// `bpf_get_stackid()` result its frame-pointer chain is stored under. +pub fn parse_stack(data: &[u8]) -> Option<(MemtrackEvent, i64)> { + let header_len = std::mem::size_of::(); + // SAFETY: the length is checked below, and the layout is the bindgen-generated C ABI struct. + let header: stack_header = if data.len() >= header_len { + unsafe { std::ptr::read_unaligned(data.as_ptr().cast()) } + } else { + warn!( + "malformed stack record: {} bytes, need at least {header_len}", + data.len() + ); + return None; + }; + + let record_len = header_len + header.copy_len as usize; + if data.len() < record_len { + warn!( + "malformed stack record: {} bytes, need {record_len}", + data.len() + ); + return None; + } + + let event = MemtrackEvent { + pid: header.pid as i32, + tid: header.tid as i32, + timestamp: header.timestamp, + addr: 0, + kind: MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: header.hash, + sp: header.sp, + regs: header.regs.reg.to_vec(), + bytes: data[header_len..record_len].to_vec(), + fp_chain: Vec::new(), + truncated: header.truncated != 0, + }), + }, + }; + + Some((event, header.stackid)) +} + +/// The frame-pointer walk recorded under `stackid`, innermost frame first. +/// Best effort: a missing chain costs the fallback for one stack, not the run. +pub fn fp_chain(stack_traces: &impl MapCore, stackid: i64) -> Vec { + let Ok(key) = u32::try_from(stackid) else { + return Vec::new(); + }; + + let value = match stack_traces.lookup(&key.to_ne_bytes(), libbpf_rs::MapFlags::ANY) { + Ok(Some(value)) => value, + Ok(None) => return Vec::new(), + Err(error) => { + warn!("Failed to read frame-pointer chain for stackid {stackid}: {error}"); + return Vec::new(); + } + }; + + // The map value is a fixed-depth array zero-padded past the last frame. + value + .chunks_exact(8) + .map(|word| u64::from_ne_bytes(word.try_into().expect("chunks_exact yields 8 bytes"))) + .take_while(|&address| address != 0) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ebpf::events::bindings::stack_regs; + + fn encode(header: stack_header, payload: &[u8]) -> Vec { + // SAFETY: The bindgen-generated C ABI struct is copied as bytes for a test fixture. + let header_bytes = unsafe { + std::slice::from_raw_parts( + (&header as *const stack_header).cast::(), + std::mem::size_of::(), + ) + }; + let mut data = header_bytes.to_vec(); + data.extend_from_slice(payload); + data + } + + fn header(copy_len: u32) -> stack_header { + stack_header { + hash: 0x0123_4567_89ab_cdef, + timestamp: 987_654_321, + stackid: -17, + sp: 0x7fff_1234_5000, + pid: 41, + tid: 42, + copy_len, + truncated: 1, + _pad: [0; 3], + regs: stack_regs { + reg: std::array::from_fn(|index| 0x1000 + index as u64), + }, + } + } + + #[test] + fn well_formed_record_round_trips_every_field() { + let header = header(5); + let payload = [1, 2, 3, 4, 5]; + + let (event, stackid) = parse_stack(&encode(header, &payload)).unwrap(); + assert_eq!(event.pid, 41); + assert_eq!(event.tid, 42); + assert_eq!(event.timestamp, 987_654_321); + assert_eq!(event.addr, 0); + assert_eq!(stackid, -17); + + let MemtrackEventKind::Stack { record } = event.kind else { + panic!("expected Stack event"); + }; + + assert_eq!(record.hash, header.hash); + assert_eq!(record.sp, header.sp); + assert_eq!(record.regs, header.regs.reg.to_vec()); + assert_eq!(record.bytes, payload); + assert!(record.fp_chain.is_empty()); + assert!(record.truncated); + } + + #[test] + fn truncated_buffer_returns_none() { + let data = vec![0; std::mem::size_of::() - 1]; + assert!(parse_stack(&data).is_none()); + } + + #[test] + fn missing_payload_returns_none() { + assert!(parse_stack(&encode(header(4), &[1, 2, 3])).is_none()); + } +} diff --git a/crates/memtrack/src/ebpf/stacks/mod.rs b/crates/memtrack/src/ebpf/stacks/mod.rs new file mode 100644 index 000000000..fc5eee33f --- /dev/null +++ b/crates/memtrack/src/ebpf/stacks/mod.rs @@ -0,0 +1,3 @@ +pub mod config; +pub mod counters; +pub mod events; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 914dfe22f..0931998f9 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -1,33 +1,76 @@ use crate::ebpf::attach_worker::AttachWorker; +use crate::ebpf::mappings::{MappingRecord, MappingSupport, resolve_mappings}; use crate::ebpf::spawn::{resume, spawn_stopped, wrap_stopped}; +use crate::ebpf::stacks::config::{clamp_copy_size, stack_copy_size_from_env}; +use crate::ebpf::stacks::counters::StackCaptureStats; use crate::ebpf::{BpfVariant, MemtrackBpf, OwnershipMaps}; use crate::prelude::*; use crate::session::Session; use parking_lot::Mutex; +use runner_shared::artifacts::MemtrackMappings; use std::os::unix::process::CommandExt; use std::process::Command; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::mpsc; pub struct Tracker { bpf: Arc>, worker: Mutex>, allocators: bool, + /// The dedup gate spans the whole BPF object, so a second session would + /// reference stack records the first one already consumed. + stacks_polled: Option, + /// Filled by the mapping poller; drained by [`Tracker::mappings`] after the + /// session is dropped, so the poller's final drain is included. + mapping_rx: Mutex>>, } impl Tracker { /// Create a new tracker. The exec-mapping watcher discovers and attaches /// allocator probes as the tracked process tree maps executable files. pub fn new() -> Result { + Self::with_stack_capture_size(stack_copy_size_from_env()) + } + + /// Like [`Tracker::new`], with allocation stack capture forced on. + pub fn new_with_stack_capture(copy_size: u32) -> Result { + Self::with_stack_capture_size(Some(clamp_copy_size(copy_size))) + } + + fn with_stack_capture_size(copy_size: Option) -> Result { let track_rmap = Self::track_rmap_from_env(); - Self::build(MemtrackBpf::new_with_rmap(track_rmap)?, true) + let mappings = MappingSupport::detect(); + + // Stacks are raw addresses: without mapped module paths nothing can + // attribute them, so capturing them would only inflate the artifact. + let copy_size = match (copy_size, mappings) { + (Some(_), MappingSupport::Unsupported) => { + warn!( + "Allocation stack capture needs in-kernel path resolution, which this host \ + cannot provide; disabling it" + ); + None + } + (copy_size, _) => copy_size, + }; + + Self::build( + MemtrackBpf::new_with_rmap(track_rmap, copy_size, mappings)?, + true, + copy_size.is_some(), + ) } /// Like [`Tracker::new`], but pinned to a specific BPF variant instead of /// the detected one. pub fn with_variant(variant: BpfVariant) -> Result { let track_rmap = Self::track_rmap_from_env(); - Self::build(MemtrackBpf::with_variant(variant, track_rmap)?, true) + Self::build( + MemtrackBpf::with_variant(variant, track_rmap, None, MappingSupport::detect())?, + true, + false, + ) } fn track_rmap_from_env() -> bool { @@ -38,15 +81,20 @@ impl Tracker { /// rmap fentries) without allocator probes; no exec-mapping watcher or /// attach worker runs. pub fn new_without_allocators_with_rmap(track_rmap: bool) -> Result { - Self::build(MemtrackBpf::new_with_rmap(track_rmap)?, false) + Self::build( + MemtrackBpf::new_with_rmap(track_rmap, None, MappingSupport::Unsupported)?, + false, + false, + ) } - fn build(mut bpf: MemtrackBpf, allocators: bool) -> Result { + fn build(mut bpf: MemtrackBpf, allocators: bool, capture_stacks: bool) -> Result { Self::bump_memlock_rlimit()?; bpf.attach_tracepoints()?; if allocators { bpf.attach_exec_watcher()?; + bpf.attach_mapping_recorder()?; } let bpf = Arc::new(Mutex::new(bpf)); @@ -60,6 +108,8 @@ impl Tracker { bpf, worker: Mutex::new(worker), allocators, + stacks_polled: capture_stacks.then(|| AtomicBool::new(false)), + mapping_rx: Mutex::new(None), }) } @@ -72,6 +122,14 @@ impl Tracker { /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). pub fn spawn(&self, cmd: &Command, uid_gid: Option<(u32, u32)>) -> Result { + let capture_stacks = match &self.stacks_polled { + Some(polled) if polled.swap(true, Ordering::Relaxed) => { + bail!("stack capture supports a single spawned command per tracker") + } + Some(_) => true, + None => false, + }; + let mut wrapped = wrap_stopped(cmd); if let Some((uid, gid)) = uid_gid { wrapped.uid(uid).gid(gid); @@ -79,6 +137,7 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; + match self.worker.lock().as_ref() { Some(worker) => worker.set_root_pid(pid), // No watcher to arm means exec mappings would be missed. @@ -87,14 +146,54 @@ impl Tracker { } let (tx, rx) = mpsc::channel(); - let poller = { + let (mapping_tx, mapping_rx) = mpsc::channel(); + let (poller, stack_poller, mapping_poller) = { let mut bpf = self.bpf.lock(); bpf.add_tracked_pid(pid)?; - bpf.poll_events_with_channel(10, tx)? + let stack_poller = capture_stacks + .then(|| bpf.poll_stacks(10, tx.clone())) + .transpose()?; + let mapping_poller = bpf + .records_mappings() + .then(|| bpf.poll_mappings_with_channel(10, mapping_tx)) + .transpose()?; + ( + bpf.poll_events_with_channel(10, tx)?, + stack_poller, + mapping_poller, + ) }; + *self.mapping_rx.lock() = Some(mapping_rx); resume(pid)?; - Ok(Session::new(child, rx, poller)) + Ok(Session::new( + child, + rx, + poller, + stack_poller, + mapping_poller, + )) + } + + /// The module mappings recorded during the run, joined with the paths the + /// kernel resolved for them. Call after dropping the session so the poller's + /// final drain is included, and before the BPF object is torn down. + pub fn mappings(&self) -> Result { + let Some(rx) = self.mapping_rx.lock().take() else { + return Ok(MemtrackMappings::default()); + }; + + let records: Vec<_> = rx.try_iter().collect(); + let paths = self.bpf.lock().mapped_paths()?; + + let dropped = self.bpf.lock().mapping_dropped_count()?; + if dropped > 0 { + warn!("{dropped} mapping records were dropped; some modules may be unresolved"); + } + + Ok(MemtrackMappings { + mappings: resolve_mappings(records, &paths), + }) } /// Enable allocator-event tracking in the BPF program. Lifetime events @@ -115,6 +214,11 @@ impl Tracker { self.bpf.lock().dropped_events_count() } + /// Per-cause counts of stack captures that were skipped or truncated. + pub fn stack_capture_stats(&self) -> Result { + self.bpf.lock().stack_capture_stats() + } + /// Only meaningful while the BPF object is alive; teardown frees the maps. pub fn ownership_maps(&self) -> Result { self.bpf.lock().ownership_maps() diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 283cff194..769d7fcac 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -159,6 +159,12 @@ fn track_command( // exec mappings mean incomplete allocator coverage). tracker.finish()?; + // Needs the BPF maps, so it has to run before teardown; the session is + // already dropped, so the poller's final drain is in the channel. + let mappings = tracker.mappings().context("Failed to collect mappings")?; + info!("Recorded {} module mappings", mappings.mappings.len()); + mappings.save_with_pid_to(out_dir, root_pid)?; + // Detach probes explicitly: the IPC thread still holds an Arc clone, so the // tracker would otherwise never be dropped before process::exit and the // kernel would close every link fd serially during exit. diff --git a/crates/memtrack/src/session.rs b/crates/memtrack/src/session.rs index 7aec33feb..551db639c 100644 --- a/crates/memtrack/src/session.rs +++ b/crates/memtrack/src/session.rs @@ -10,6 +10,8 @@ pub struct Session { child: Child, events: Option>, _poller: RingBufferPoller, + _stack_poller: Option, + _mapping_poller: Option, } impl Session { @@ -17,11 +19,15 @@ impl Session { child: Child, events: Receiver, poller: RingBufferPoller, + stack_poller: Option, + mapping_poller: Option, ) -> Self { Self { child, events: Some(events), _poller: poller, + _stack_poller: stack_poller, + _mapping_poller: mapping_poller, } } diff --git a/crates/memtrack/testdata/stack_paths.c b/crates/memtrack/testdata/stack_paths.c new file mode 100644 index 000000000..0cee7f437 --- /dev/null +++ b/crates/memtrack/testdata/stack_paths.c @@ -0,0 +1,46 @@ +#include +#include + +static volatile void *escaped_pointer; +static volatile unsigned int remaining_a = 50; +static volatile unsigned int remaining_b = 50; + +__attribute__((noinline)) static void path_a_inner(void) { + void *pointer = malloc(64); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_a(void) { + while (remaining_a != 0) { + path_a_inner(); + --remaining_a; + } +} + +__attribute__((noinline)) static void path_b_inner(void) { + void *pointer = malloc(192); + escaped_pointer = pointer; + free(pointer); +} + +__attribute__((noinline)) static void path_b(void) { + while (remaining_b != 0) { + path_b_inner(); + --remaining_b; + } +} + +int main(void) { + void *marker_before = malloc(0xC0D59EED); + escaped_pointer = marker_before; + free(marker_before); + + path_a(); + path_b(); + + void *marker_after = malloc(0xC0D59EED); + escaped_pointer = marker_after; + free(marker_after); + return 0; +} diff --git a/crates/memtrack/tests/dlopen_tests.rs b/crates/memtrack/tests/dlopen_tests.rs index cbabfeafc..dfca50444 100644 --- a/crates/memtrack/tests/dlopen_tests.rs +++ b/crates/memtrack/tests/dlopen_tests.rs @@ -67,18 +67,19 @@ fn test_dlopen_allocator() -> Result<(), Box> { let malloc_addrs: HashSet = events .iter() .filter_map(|e| match e.kind { - MemtrackEventKind::Malloc { size: 4242 } => Some(e.addr), + MemtrackEventKind::Malloc { size: 4242, .. } => Some(e.addr), _ => None, }) .collect(); let malloc_count = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let free_count = events .iter() .filter(|e| { - matches!(e.kind, MemtrackEventKind::Free) && malloc_addrs.contains(&e.addr) + matches!(e.kind, MemtrackEventKind::Free { .. }) + && malloc_addrs.contains(&e.addr) }) .count(); @@ -125,11 +126,11 @@ fn test_thread_dlopen() -> Result<(), Box> { |events| { let m4242 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4242, .. })) .count(); let m4243 = events .iter() - .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243 })) + .filter(|e| matches!(e.kind, MemtrackEventKind::Malloc { size: 4243, .. })) .count(); assert_eq!(m4242, 100, "expected 100 mi_malloc(4242) events"); diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index 499d74c53..d307b6260 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -25,7 +25,7 @@ macro_rules! assert_events_snapshot { let formatted_events: Vec = $events .iter() .filter(|e| { - // RSS and lifecycle events are asserted by dedicated tests, not snapshots. + // RSS, lifecycle and stack events are asserted by dedicated tests, not snapshots. !matches!( e.kind, MemtrackEventKind::Rss { .. } @@ -33,6 +33,7 @@ macro_rules! assert_events_snapshot { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -88,11 +89,14 @@ macro_rules! assert_events_with_marker_for_each_variant { }; } -/// An event's kind and size, without the addresses that differ between runs of -/// the same workload. `Realloc` needs spelling out since its `Debug` includes the -/// old address. +/// An event's kind and size, without the addresses or stack identities that +/// differ between runs of the same workload. pub fn describe_kind(kind: &MemtrackEventKind) -> String { match kind { + MemtrackEventKind::Free { .. } => "Free".to_string(), + MemtrackEventKind::Malloc { size, .. } => format!("Malloc {{ size: {size} }}"), + MemtrackEventKind::Calloc { size, .. } => format!("Calloc {{ size: {size} }}"), + MemtrackEventKind::AlignedAlloc { size, .. } => format!("AlignedAlloc {{ size: {size} }}"), MemtrackEventKind::Realloc { size, .. } => format!("Realloc {{ size: {size} }}"), other => format!("{other:?}"), } @@ -106,7 +110,7 @@ pub fn between_markers(events: &[Event]) -> Vec { const MARKER: u64 = 0xC0D5_9EED; let is_marker = - |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size } if size == MARKER); + |e: &&Event| matches!(e.kind, MemtrackEventKind::Malloc { size, .. } if size == MARKER); events .iter() @@ -122,6 +126,7 @@ pub fn between_markers(events: &[Event]) -> Vec { | MemtrackEventKind::Fork { .. } | MemtrackEventKind::Exec | MemtrackEventKind::Exit + | MemtrackEventKind::Stack { .. } ) }) .sorted_by_key(|e| e.timestamp) @@ -222,6 +227,11 @@ pub fn track_command_with_rmap_maps( Ok((events, maps, std::thread::spawn(move || drop(tracker)))) } +/// Track a command with allocation stack capture enabled, returning its events. +pub fn track_command_with_stacks(command: Command, copy_size: u32) -> TrackResult { + track_command_with_tracker(command, Tracker::new_with_stack_capture(copy_size)?) +} + /// Track a command with rmap hooks and snapshot the ownership maps at a /// fixture-signalled checkpoint. /// @@ -266,7 +276,7 @@ fn event_profile(events: &[Event]) -> EventProfile { if !matches!( event.kind, MemtrackEventKind::Malloc { .. } - | MemtrackEventKind::Free + | MemtrackEventKind::Free { .. } | MemtrackEventKind::Calloc { .. } | MemtrackEventKind::Realloc { .. } | MemtrackEventKind::AlignedAlloc { .. } diff --git a/crates/memtrack/tests/stack_tests.rs b/crates/memtrack/tests/stack_tests.rs new file mode 100644 index 000000000..7a95cd6ca --- /dev/null +++ b/crates/memtrack/tests/stack_tests.rs @@ -0,0 +1,240 @@ +#[macro_use] +mod shared; + +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use std::collections::HashSet; +use std::process::Command; +use tempfile::TempDir; + +const COPY_SIZE: u32 = memtrack::DEFAULT_STACK_COPY_SIZE; + +fn compile_fixture( + name: &str, + temp_dir: &TempDir, +) -> Result> { + shared::compile_c_source( + include_str!("../testdata/stack_paths.c"), + name, + temp_dir.path(), + ) +} +fn require_mapping_support() -> bool { + if memtrack::MappingSupport::detect() == memtrack::MappingSupport::Unsupported { + eprintln!("skipping stack capture test: mapping support is unavailable"); + return false; + } + true +} + +/// The stack identity carried by each allocation and deallocation event that has one. +fn event_hashes(events: &[MemtrackEvent]) -> Vec { + events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Malloc { stack_hash, .. } + | MemtrackEventKind::Calloc { stack_hash, .. } + | MemtrackEventKind::AlignedAlloc { stack_hash, .. } + | MemtrackEventKind::Realloc { stack_hash, .. } + | MemtrackEventKind::Free { stack_hash } => (stack_hash != 0).then_some(stack_hash), + _ => None, + }) + .collect() +} + +fn record_hashes(events: &[MemtrackEvent]) -> HashSet { + events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record } => Some(record.hash), + _ => None, + }) + .collect() +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn distinct_call_paths_get_distinct_stacks() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + + let records: Vec<_> = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record: r } => { + Some((r.hash, r.sp, &r.regs, &r.bytes, r.truncated)) + } + _ => None, + }) + .collect(); + + assert!( + records.len() >= 2, + "expected at least two stack records, got {} ({} events)", + records.len(), + events.len() + ); + + let hashes = record_hashes(&events); + assert_eq!( + hashes.len(), + records.len(), + "stack records must be deduplicated by unique hash" + ); + + for (hash, sp, regs, bytes, truncated) in &records { + assert_ne!(*sp, 0, "record {hash:#x} has no stack pointer"); + assert_eq!(regs.len(), 33, "record {hash:#x} must carry 33 registers"); + assert!( + !bytes.is_empty() && bytes.len() % 512 == 0 && bytes.len() <= COPY_SIZE as usize, + "record {hash:#x} must hold whole 512-byte chunks within the budget, got {}", + bytes.len() + ); + assert_eq!( + *truncated, + bytes.len() == COPY_SIZE as usize, + "record {hash:#x} may only be flagged truncated when it filled the budget" + ); + } + + let carried = event_hashes(&events); + assert!( + !carried.is_empty(), + "expected events carrying a captured stack hash" + ); + assert!( + carried.iter().all(|hash| hashes.contains(hash)), + "every non-zero stack_hash must have a matching stack record" + ); + + // The fixture frees every allocation, so both sides must report identities. + assert!( + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Free { stack_hash } if stack_hash != 0)), + "free events must carry their own stack identity" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn dedup_collapses_repeated_call_paths() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_dedup", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), COPY_SIZE)?; + + let carried = event_hashes(&events); + let records = record_hashes(&events); + assert!( + carried.len() > records.len(), + "expected repeated call paths to deduplicate raw stacks: {} stack-bearing events across {} unique stacks ({} total events)", + carried.len(), + records.len(), + events.len() + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// The largest budget stresses the verifier hardest: the copy loop and its +/// unrolled per-chunk hash both scale with the configured size, so a program +/// that loads at the default can still exceed the instruction limit here. +/// It is also the only budget at which nothing can be budget-limited, because +/// the stack mapping always ends first. +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn max_copy_budget_loads_and_captures_whole_stacks() -> Result<(), Box> { + if !require_mapping_support() { + return Ok(()); + } + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_max", &temp_dir)?; + let (events, thread_handle) = + shared::track_command_with_stacks(Command::new(&binary), u32::MAX)?; + + let truncated: Vec<_> = events + .iter() + .filter_map(|e| match &e.kind { + MemtrackEventKind::Stack { record: r } if r.truncated => Some(r.hash), + _ => None, + }) + .collect(); + + assert!( + !record_hashes(&events).is_empty(), + "expected stack records at the maximum copy budget" + ); + assert!( + truncated.is_empty(), + "no capture can be budget-limited at the maximum budget: {truncated:#x?}" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} + +/// Restores the capture toggle on drop so a failing assertion cannot leak the +/// override into later tests (the suite runs single-threaded). +struct DisableCaptureGuard; + +impl DisableCaptureGuard { + fn set() -> Self { + // SAFETY: tests run with --test-threads 1, so no concurrent env access. + unsafe { std::env::set_var("CODSPEED_MEMTRACK_CAPTURE_STACKS", "0") }; + Self + } +} + +impl Drop for DisableCaptureGuard { + fn drop(&mut self) { + // SAFETY: see `set`. + unsafe { std::env::remove_var("CODSPEED_MEMTRACK_CAPTURE_STACKS") }; + } +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test_log::test] +fn explicit_disable_suppresses_stack_capture() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = compile_fixture("stack_paths_disabled", &temp_dir)?; + let _guard = DisableCaptureGuard::set(); + let (events, thread_handle) = shared::track_binary(&binary)?; + + assert!( + events + .iter() + .any(|e| matches!(e.kind, MemtrackEventKind::Malloc { .. })), + "disabled capture must still report allocation events" + ); + assert!( + record_hashes(&events).is_empty(), + "disabled capture must emit zero stack records" + ); + assert!( + event_hashes(&events).is_empty(), + "disabled capture must leave stack_hash zero on every event" + ); + + thread_handle + .join() + .expect("tracker teardown thread panicked"); + Ok(()) +} diff --git a/crates/runner-shared/Cargo.toml b/crates/runner-shared/Cargo.toml index 8b8f6ab97..9c3c9f188 100644 --- a/crates/runner-shared/Cargo.toml +++ b/crates/runner-shared/Cargo.toml @@ -7,6 +7,7 @@ edition = "2024" [dependencies] anyhow = { workspace = true } serde = { workspace = true } +serde_bytes = "0.11" serde_json = { workspace = true } # Pinned to 1.x: 2.0 changes the wire format and serde integration bincode = "1.3" diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index a6c610e8e..3a432866f 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -14,14 +14,24 @@ fn generate_events(n: usize) -> Vec { for _ in 0..n { let size = rng.gen_range(8..8192); let kind = match rng.gen_range(0..10) { - 0 => MemtrackEventKind::Malloc { size }, - 1 => MemtrackEventKind::Free, + 0 => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, + 1 => MemtrackEventKind::Free { stack_hash: 0 }, 2 => MemtrackEventKind::Realloc { old_addr: Some(rng.r#gen()), size, + stack_hash: 0, + }, + 3 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, + 4 => MemtrackEventKind::AlignedAlloc { + size, + stack_hash: 0, }, - 3 => MemtrackEventKind::Calloc { size }, - 4 => MemtrackEventKind::AlignedAlloc { size }, 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, @@ -90,12 +100,18 @@ fn generate_realistic_events(n: usize) -> Vec { addr }); let kind = match rng.gen_range(0..20) { - 0 => MemtrackEventKind::Calloc { size }, + 0 => MemtrackEventKind::Calloc { + size, + stack_hash: 0, + }, 1 => MemtrackEventKind::Mmap { size }, - _ => MemtrackEventKind::Malloc { size }, + _ => MemtrackEventKind::Malloc { + size, + stack_hash: 0, + }, }; - if let MemtrackEventKind::Mmap { size } = kind { - live_mmap.push((addr, size)); + if let MemtrackEventKind::Mmap { size } = &kind { + live_mmap.push((addr, *size)); } else { live_heap.push(addr); } @@ -105,7 +121,7 @@ fn generate_realistic_events(n: usize) -> Vec { if idx < live_heap.len() { let addr = live_heap.swap_remove(idx); free_list.push(addr); - (addr, MemtrackEventKind::Free) + (addr, MemtrackEventKind::Free { stack_hash: 0 }) } else { let (addr, size) = live_mmap.swap_remove(idx - live_heap.len()); free_list.push(addr); @@ -127,6 +143,7 @@ fn generate_realistic_events(n: usize) -> Vec { MemtrackEventKind::Realloc { old_addr: Some(old_addr), size, + stack_hash: 0, }, ) }; @@ -150,7 +167,7 @@ fn encode_events_realistic(bencher: Bencher, n_workers: usize) { bencher.bench_local(|| { let mut output = Vec::new(); - encode_events(events.iter().copied(), &mut output, n_workers).unwrap(); + encode_events(events.iter().cloned(), &mut output, n_workers).unwrap(); output }); } diff --git a/crates/runner-shared/src/artifacts/memtrack/mappings.rs b/crates/runner-shared/src/artifacts/memtrack/mappings.rs new file mode 100644 index 000000000..b271dfee4 --- /dev/null +++ b/crates/runner-shared/src/artifacts/memtrack/mappings.rs @@ -0,0 +1,37 @@ +use libc::pid_t; +use serde::{Deserialize, Serialize}; +use std::ops::Range; + +/// The file-backed mappings the tracked process tree loaded, recorded as they +/// happened. Companion to the event stream: allocation stacks are raw +/// addresses, and these are what turns them back into modules. +/// +/// Kept out of the event stream so a consumer that only needs the module set +/// does not have to decode millions of allocation events. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct MemtrackMappings { + pub mappings: Vec, +} + +impl super::super::ArtifactExt for MemtrackMappings {} + +/// One executable mapping of one file into one process, as `PERF_RECORD_MMAP2` +/// would describe it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProcessMapping { + pub pid: pid_t, + /// Resolved in-kernel at mmap time, so it is correct for the mapping + /// process's mount namespace even if the process is already gone. + pub path: String, + /// Kernel `s_dev` encoding: `(major << 20) | minor`. With `ino`, proves at + /// analysis time that the path still names the file that was mapped. + pub dev: u64, + pub ino: u64, + /// Offset of the mapping's first byte in the file. In bytes, matching + /// `PERF_RECORD_MMAP2`'s `pgoff` and the load-bias computation. + pub file_offset: u64, + pub avma_range: Range, + /// CLOCK_MONOTONIC nanoseconds, the same clock the events carry. The + /// mapping is valid from here until a later mapping covers the range. + pub timestamp: u64, +} diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index b082a7c67..0ba0ced47 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -2,9 +2,11 @@ use libc::pid_t; use serde::{Deserialize, Serialize}; use std::io::{BufReader, Read, Write}; +mod mappings; mod pipeline; mod writer; +pub use mappings::*; pub use pipeline::*; pub use writer::*; @@ -41,7 +43,7 @@ impl MemtrackArtifact { } } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct MemtrackEvent { pub pid: pid_t, pub tid: pid_t, @@ -51,23 +53,34 @@ pub struct MemtrackEvent { pub kind: MemtrackEventKind, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "type")] pub enum MemtrackEventKind { Malloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, + }, + Free { + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, - Free, Realloc { #[serde(default, skip_serializing_if = "Option::is_none")] old_addr: Option, size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Calloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, AlignedAlloc { size: u64, + #[serde(default, skip_serializing_if = "is_zero")] + stack_hash: u64, }, Mmap { size: u64, @@ -91,6 +104,31 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + + Stack { + #[serde(flatten)] + record: Box, + }, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StackRecord { + pub hash: u64, + /// User stack pointer the copy starts at. + pub sp: u64, + /// Registers by DWARF number for the capturing architecture; 33 entries on x86_64. + pub regs: Vec, + /// Raw stack bytes read upward from `sp`. + #[serde(with = "serde_bytes")] + pub bytes: Vec, + /// In-kernel frame-pointer walk, innermost first; empty when unavailable. + pub fp_chain: Vec, + /// The copy filled its budget, so stack above it was not captured. + pub truncated: bool, +} + +fn is_zero(value: &u64) -> bool { + *value == 0 } pub struct MemtrackEventStream { @@ -120,14 +158,17 @@ mod tests { tid: 11, timestamp: 100, addr: 0x10, - kind: MemtrackEventKind::Malloc { size: 64 }, + kind: MemtrackEventKind::Malloc { + size: 64, + stack_hash: 0, + }, }, MemtrackEvent { pid: 1, tid: 12, timestamp: 200, addr: 0x20, - kind: MemtrackEventKind::Free, + kind: MemtrackEventKind::Free { stack_hash: 0 }, }, MemtrackEvent { pid: 1, @@ -167,21 +208,47 @@ mod tests { } let kinds = [ - MemtrackEventKind::Malloc { size: 7 }, - MemtrackEventKind::Free, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0, + }, + MemtrackEventKind::Malloc { + size: 7, + stack_hash: 0xCAFE_BABE, + }, + MemtrackEventKind::Free { stack_hash: 0 }, + MemtrackEventKind::Free { stack_hash: 0xFEED }, MemtrackEventKind::Realloc { old_addr: Some(0x1000), size: 42, + stack_hash: 0, }, MemtrackEventKind::Realloc { old_addr: None, size: 42, + stack_hash: 0x1234, + }, + MemtrackEventKind::Calloc { + size: 9, + stack_hash: 0, + }, + MemtrackEventKind::AlignedAlloc { + size: 9, + stack_hash: 0, }, - MemtrackEventKind::Calloc { size: 9 }, - MemtrackEventKind::AlignedAlloc { size: 9 }, MemtrackEventKind::Mmap { size: 9 }, MemtrackEventKind::Munmap { size: 9 }, MemtrackEventKind::Brk { size: 9 }, + MemtrackEventKind::Stack { + record: Box::new(StackRecord { + hash: 0xDEAD_BEEF, + sp: 0x7FFF_0000, + regs: vec![0; 33], + bytes: vec![1, 2, 3, 4], + fp_chain: vec![0x1000, 0x2000], + truncated: false, + }), + }, ]; for kind in kinds { @@ -190,7 +257,7 @@ mod tests { tid: 42, timestamp: 0xDEAD, addr: 0xBEEF, - kind, + kind: kind.clone(), }; let shadow = Shadow { pid: -7, @@ -215,7 +282,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect(); @@ -265,7 +335,8 @@ mod tests { event.kind, MemtrackEventKind::Realloc { old_addr: None, - size: 42 + size: 42, + stack_hash: 0, } )); diff --git a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs index c47b3aed9..8cac46f05 100644 --- a/crates/runner-shared/src/artifacts/memtrack/pipeline.rs +++ b/crates/runner-shared/src/artifacts/memtrack/pipeline.rs @@ -94,7 +94,10 @@ mod tests { tid: 1, timestamp: i, addr: i, - kind: MemtrackEventKind::Malloc { size: i }, + kind: MemtrackEventKind::Malloc { + size: i, + stack_hash: 0, + }, }) .collect() } diff --git a/crates/runner-shared/src/lib.rs b/crates/runner-shared/src/lib.rs index 61e804de7..2cdc7d5a6 100644 --- a/crates/runner-shared/src/lib.rs +++ b/crates/runner-shared/src/lib.rs @@ -4,5 +4,6 @@ pub mod fifo; pub mod metadata; pub mod module_symbols; pub mod perf_event; +pub mod serde_pid_map; pub mod unwind_data; pub mod walltime_results; diff --git a/crates/runner-shared/src/metadata.rs b/crates/runner-shared/src/metadata.rs index 7a2c7c89b..654ae298d 100644 --- a/crates/runner-shared/src/metadata.rs +++ b/crates/runner-shared/src/metadata.rs @@ -11,35 +11,43 @@ use crate::fifo::MarkerType; use crate::module_symbols::MappedProcessModuleSymbols; use crate::unwind_data::MappedProcessUnwindData; +/// The per-profile module artifacts: the deduplicated debug info, unwind data +/// and symbol tables extracted from the ELF modules the profiled processes +/// mapped, plus the per-pid references into them. +/// +/// Flattened into every metadata format, so all profiling modes describe their +/// modules identically. #[derive(Serialize, Deserialize, Default)] -pub struct WalltimeMetadata { - /// The version of this metadata format. - pub version: u64, - - /// Name and version of the integration - pub integration: (String, String), - - /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias - #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub ignored_modules_by_pid: HashMap>, - +pub struct ModuleArtifacts { /// Deduplicated debug info entries, keyed by semantic key #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub debug_info: HashMap, /// Per-pid debug info references, mapping PID to mounted modules' debug info /// Referenced by `path_keys` that point to the deduplicated `debug_info` entries. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_debug_info_by_pid: HashMap>, /// Per-pid unwind data references, mapping PID to mounted modules' unwind data /// Referenced by `path_keys` that point to the deduplicated `unwind_data` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_unwind_data_by_pid: HashMap>, /// Per-pid symbol references, mapping PID to its mounted modules' symbols /// Referenced by `path_keys` that point to the deduplicated `symbols.map` files on disk. - #[serde(default, skip_serializing_if = "HashMap::is_empty")] + #[serde( + default, + skip_serializing_if = "HashMap::is_empty", + with = "crate::serde_pid_map" + )] pub mapped_process_module_symbols: HashMap>, /// Mapping from semantic `path_key` to original binary path on host disk @@ -49,6 +57,22 @@ pub struct WalltimeMetadata { /// Until now, only kept for traceability, if we ever need to reconstruct the original paths from the keys #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub path_key_to_path: HashMap, +} + +#[derive(Serialize, Deserialize, Default)] +pub struct WalltimeMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + /// Per-pid modules that should be ignored, with runtime address ranges derived from symbol bounds + load bias + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub ignored_modules_by_pid: HashMap>, + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, // Deprecated fields below are kept for backward compatibility, since this struct is used in // the parser and older versions of the runner still generate them @@ -85,3 +109,142 @@ impl WalltimeMetadata { Ok(()) } } + +/// Companion to the memtrack event stream: the modules its allocation stacks +/// resolve against. Memory mode records benchmark boundaries in +/// `ExecutionTimestamps`, so unlike [`WalltimeMetadata`] it carries no markers. +#[derive(Serialize, Deserialize, Default)] +pub struct MemtrackMetadata { + /// The version of this metadata format. + pub version: u64, + + /// Name and version of the integration + pub integration: (String, String), + + #[serde(flatten)] + pub artifacts: ModuleArtifacts, +} + +impl MemtrackMetadata { + pub fn from_reader(reader: R) -> anyhow::Result { + serde_json::from_reader(reader).context("Could not parse memtrack metadata from JSON") + } + + pub fn save_to>(&self, path: P) -> anyhow::Result<()> { + let file = std::fs::File::create(path.as_ref().join("memtrack.metadata"))?; + const BUFFER_SIZE: usize = 256 * 1024 /* 256 KB */; + + let writer = BufWriter::with_capacity(BUFFER_SIZE, file); + serde_json::to_writer(writer, self)?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Captured from the flat `WalltimeMetadata` that predates + /// [`ModuleArtifacts`]: flattening must not move a single byte, since the + /// parser reads this format from runners of every version. + const WALLTIME_JSON: &str = r#"{"version":7,"integration":["codspeed-rust","4.2.0"],"ignored_modules_by_pid":{"42":[["/lib/libpython.so",4096,8192]]},"debug_info":{"0__libc.so.6":{"object_path":"/lib/libc.so.6","addr_bounds":[4096,36864],"load_bias":4096,"debug_infos":[{"addr":4352,"size":32,"name":"malloc","file":"malloc.c","line":11}]}},"mapped_process_debug_info_by_pid":{"42":[{"debug_info_key":"0__libc.so.6","load_bias":4096}]},"mapped_process_unwind_data_by_pid":{"42":[{"unwind_data_key":"0__libc.so.6","timestamp":1234,"avma_range":{"start":4096,"end":36864},"base_avma":4096}]},"mapped_process_module_symbols":{"42":[{"perf_map_key":"0__libc.so.6","load_bias":4096}]},"path_key_to_path":{"0__libc.so.6":"/lib/libc.so.6"},"uri_by_ts":[[1,"bench::a"]],"ignored_modules":[],"markers":[]}"#; + + fn populated_artifacts() -> ModuleArtifacts { + ModuleArtifacts { + debug_info: HashMap::from([( + "0__libc.so.6".to_string(), + ModuleDebugInfo { + object_path: "/lib/libc.so.6".to_string(), + addr_bounds: (0x1000, 0x9000), + load_bias: 0x1000, + debug_infos: vec![crate::debug_info::DebugInfo { + addr: 0x1100, + size: 0x20, + name: "malloc".to_string(), + file: "malloc.c".to_string(), + line: Some(11), + }], + }, + )]), + mapped_process_debug_info_by_pid: HashMap::from([( + 42, + vec![MappedProcessDebugInfo { + debug_info_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + mapped_process_unwind_data_by_pid: HashMap::from([( + 42, + vec![MappedProcessUnwindData { + unwind_data_key: "0__libc.so.6".to_string(), + inner: crate::unwind_data::ProcessUnwindData { + timestamp: Some(1234), + avma_range: 0x1000..0x9000, + base_avma: 0x1000, + }, + }], + )]), + mapped_process_module_symbols: HashMap::from([( + 42, + vec![crate::module_symbols::MappedProcessModuleSymbols { + perf_map_key: "0__libc.so.6".to_string(), + load_bias: 0x1000, + }], + )]), + path_key_to_path: HashMap::from([( + "0__libc.so.6".to_string(), + PathBuf::from("/lib/libc.so.6"), + )]), + } + } + + #[test] + fn walltime_metadata_serialization_is_unchanged_by_flattening() { + #[allow(deprecated)] + let metadata = WalltimeMetadata { + version: 7, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + ignored_modules_by_pid: HashMap::from([( + 42, + vec![("/lib/libpython.so".to_string(), 0x1000, 0x2000)], + )]), + artifacts: populated_artifacts(), + uri_by_ts: vec![(1, "bench::a".to_string())], + ignored_modules: vec![], + markers: vec![], + debug_info_by_pid: HashMap::new(), + }; + + assert_eq!(serde_json::to_string(&metadata).unwrap(), WALLTIME_JSON); + } + + #[test] + fn walltime_metadata_round_trips_through_the_flattened_fields() { + let parsed = WalltimeMetadata::from_reader(WALLTIME_JSON.as_bytes()).unwrap(); + + assert_eq!(parsed.artifacts.path_key_to_path.len(), 1); + assert_eq!( + parsed.artifacts.mapped_process_unwind_data_by_pid[&42].len(), + 1 + ); + assert_eq!(serde_json::to_string(&parsed).unwrap(), WALLTIME_JSON); + } + + #[test] + fn memtrack_metadata_round_trips() { + let metadata = MemtrackMetadata { + version: 1, + integration: ("codspeed-rust".to_string(), "4.2.0".to_string()), + artifacts: populated_artifacts(), + }; + + let json = serde_json::to_string(&metadata).unwrap(); + let parsed = MemtrackMetadata::from_reader(json.as_bytes()).unwrap(); + + assert_eq!(serde_json::to_string(&parsed).unwrap(), json); + assert_eq!( + parsed.artifacts.mapped_process_module_symbols[&42][0].perf_map_key, + "0__libc.so.6" + ); + } +} diff --git a/crates/runner-shared/src/serde_pid_map.rs b/crates/runner-shared/src/serde_pid_map.rs new file mode 100644 index 000000000..fa6fbb9dc --- /dev/null +++ b/crates/runner-shared/src/serde_pid_map.rs @@ -0,0 +1,36 @@ +//! `#[serde(with = ...)]` support for pid-keyed maps. +//! +//! JSON object keys are always strings. serde_json's direct deserializer +//! special-cases that and parses integer map keys, but a `#[serde(flatten)]` +//! field is buffered into serde's internal `Content` first, and that path has no +//! such special case — an `i32` key then fails with `invalid type: string`. So +//! the keys are read as strings and parsed here, which works on both paths. + +use libc::pid_t; +use serde::de::{Deserializer, Error}; +use serde::{Deserialize, Serialize, Serializer}; +use std::collections::HashMap; + +pub fn serialize(map: &HashMap, serializer: S) -> Result +where + V: Serialize, + S: Serializer, +{ + map.serialize(serializer) +} + +pub fn deserialize<'de, V, D>(deserializer: D) -> Result, D::Error> +where + V: Deserialize<'de>, + D: Deserializer<'de>, +{ + HashMap::::deserialize(deserializer)? + .into_iter() + .map(|(key, value)| { + let pid = key + .parse::() + .map_err(|_| D::Error::custom(format!("invalid pid key: {key}")))?; + Ok((pid, value)) + }) + .collect() +} diff --git a/src/executor/helpers/debug_file.rs b/src/executor/helpers/debug_file.rs index 0619bed2b..e3bbd5069 100644 --- a/src/executor/helpers/debug_file.rs +++ b/src/executor/helpers/debug_file.rs @@ -12,6 +12,12 @@ use std::path::{Path, PathBuf}; /// /// [Separate Debug Files]: https://sourceware.org/gdb/current/onlinedocs/gdb.html/Separate-Debug-Files.html pub fn find_debug_file(object: &object::File, binary_path: &Path) -> Option { + if let Some(dir) = binary_path.parent() { + if let Some(path) = find_debug_file_in(object, binary_path, dir) { + return Some(path); + } + } + ["/usr/lib/debug", "/run/current-system/sw/lib/debug"] .iter() .map(Path::new) diff --git a/src/executor/memory/executor.rs b/src/executor/memory/executor.rs index b8c9a3985..d0f44bba8 100644 --- a/src/executor/memory/executor.rs +++ b/src/executor/memory/executor.rs @@ -8,6 +8,7 @@ use crate::executor::helpers::get_bench_command::get_bench_command; use crate::executor::helpers::run_command_with_log_pipe::run_command_with_log_pipe_and_callback; use crate::executor::helpers::run_with_env::prefix_command_with_env; use crate::executor::helpers::run_with_sudo::is_root_user; +use crate::executor::memory::module_artifacts::save_module_artifacts; use crate::executor::memory::tunables::MemoryTunables; use crate::executor::shared::fifo::RunnerFifo; use crate::executor::{ExecutionContext, Executor}; @@ -24,6 +25,7 @@ use runner_shared::artifacts::{ArtifactExt, ExecutionTimestamps}; use runner_shared::fifo::Command as FifoCommand; use runner_shared::fifo::IntegrationMode; use semver::Version; +use std::cell::RefCell; use std::fs::canonicalize; use std::path::Path; use std::rc::Rc; @@ -163,7 +165,8 @@ impl Executor for MemoryExecutor { let _tunables = MemoryTunables::apply(); // Create the results/ directory inside the profile folder to avoid having memtrack create it with wrong permissions - std::fs::create_dir_all(execution_context.profile_folder.join("results"))?; + let results_folder = execution_context.profile_folder.join("results"); + std::fs::create_dir_all(&results_folder)?; Self::ensure_privileges()?; @@ -172,16 +175,19 @@ impl Executor for MemoryExecutor { debug!("cmd: {cmd:?}"); let runner_fifo = RunnerFifo::new()?; - let on_process_started = |mut child: std::process::Child| async move { - let (marker_result, exit_status) = - Self::handle_fifo(runner_fifo, ipc, &mut child).await?; - - // Directly write to the profile folder, to avoid having to define another field - marker_result - .save_to(execution_context.profile_folder.join("results")) - .unwrap(); - - Ok(exit_status) + let integration = Rc::new(RefCell::new(None)); + let on_process_started = { + let integration = integration.clone(); + |mut child: std::process::Child| async move { + let (marker_result, fifo_data, exit_status) = + Self::handle_fifo(runner_fifo, ipc, &mut child).await?; + *integration.borrow_mut() = fifo_data.integration; + + // Directly write to the profile folder, to avoid having to define another field + marker_result.save_to(&results_folder).unwrap(); + + Ok(exit_status) + } }; let status = run_command_with_log_pipe_and_callback(cmd, on_process_started).await?; @@ -191,6 +197,20 @@ impl Executor for MemoryExecutor { bail!("failed to execute memory tracker process: {status}"); } + // Without an integration no benchmark ran, which `teardown` reports. + if let Some(integration) = integration.borrow_mut().take() { + let results_folder = execution_context.profile_folder.join("results"); + if let Err(e) = save_module_artifacts( + &execution_context.profile_folder, + &results_folder, + integration, + ) { + // The memory results are complete without them; only offline + // stack attribution is lost. + error!("Failed to save memtrack module artifacts: {e:#}"); + } + } + Ok(()) } @@ -228,7 +248,11 @@ impl MemoryExecutor { mut runner_fifo: RunnerFifo, ipc: MemtrackIpcServer, child: &mut std::process::Child, - ) -> anyhow::Result<(ExecutionTimestamps, std::process::ExitStatus)> { + ) -> anyhow::Result<( + ExecutionTimestamps, + crate::executor::shared::fifo::FifoBenchmarkData, + std::process::ExitStatus, + )> { // Accept the IPC connection from memtrack and get the sender it sends us // Use a timeout to prevent hanging if the process doesn't start properly // https://github.com/servo/ipc-channel/issues/261 @@ -300,9 +324,9 @@ impl MemoryExecutor { Ok(None) }; - let (marker_result, _, exit_status) = + let (marker_result, fifo_data, exit_status) = runner_fifo.handle_fifo_messages(child, on_cmd).await?; - Ok((marker_result, exit_status)) + Ok((marker_result, fifo_data, exit_status)) } } diff --git a/src/executor/memory/mod.rs b/src/executor/memory/mod.rs index 2d17547d1..9f48a81ab 100644 --- a/src/executor/memory/mod.rs +++ b/src/executor/memory/mod.rs @@ -1,3 +1,4 @@ pub mod executor; +pub(crate) mod module_artifacts; pub(crate) mod setup; pub(crate) mod tunables; diff --git a/src/executor/memory/module_artifacts.rs b/src/executor/memory/module_artifacts.rs new file mode 100644 index 000000000..235dc1243 --- /dev/null +++ b/src/executor/memory/module_artifacts.rs @@ -0,0 +1,275 @@ +use crate::executor::shared::module_artifacts::loaded_module::LoadedModule; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::save_artifacts::save_artifacts; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; +use crate::prelude::*; +use runner_shared::artifacts::{ArtifactExt, MemtrackMappings, ProcessMapping}; +use runner_shared::metadata::MemtrackMetadata; +use std::collections::HashMap; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; + +/// The version of the memtrack metadata format. +const MEMTRACK_METADATA_CURRENT_VERSION: u64 = 1; + +/// Turn the mappings memtrack recorded into the artifacts an offline unwinder +/// needs: the deduplicated `unwind_data`/`symbols.map` files, plus the +/// `memtrack.metadata` referencing them per pid. +/// +/// `results_folder` is where memtrack wrote its artifacts; the keyed files and +/// the metadata land in `profile_folder`, next to walltime's equivalents. +pub fn save_module_artifacts( + profile_folder: &Path, + results_folder: &Path, + integration: (String, String), +) -> Result<()> { + let mappings = read_mappings(results_folder)?; + if mappings.is_empty() { + debug!("No module mappings recorded, skipping memtrack module artifacts"); + return Ok(()); + } + + let loaded_modules = loaded_modules_from_mappings(&mappings); + debug!( + "Extracting artifacts for {} modules from {} mappings", + loaded_modules.len(), + mappings.len() + ); + + let saved = save_artifacts(profile_folder, &loaded_modules, &HashMap::new()); + MemtrackMetadata { + version: MEMTRACK_METADATA_CURRENT_VERSION, + integration, + artifacts: saved.artifacts, + } + .save_to(profile_folder) +} + +/// Read every mapping artifact in the folder. One is written per tracked root +/// process, so a run with several of them contributes several files. +fn read_mappings(results_folder: &Path) -> Result> { + let suffix = format!(".{}.msgpack", MemtrackMappings::name()); + + let mut mappings = Vec::new(); + for entry in std::fs::read_dir(results_folder)?.filter_map(Result::ok) { + if !entry.file_name().to_string_lossy().ends_with(&suffix) { + continue; + } + + let file = std::fs::File::open(entry.path())?; + let artifact = MemtrackMappings::decode_from_reader(file) + .with_context(|| format!("Failed to decode {:?}", entry.path()))?; + mappings.extend(artifact.mappings); + } + Ok(mappings) +} + +fn loaded_modules_from_mappings(mappings: &[ProcessMapping]) -> HashMap { + let mut loaded_modules = HashMap::::new(); + + for mapping in mappings { + let path = PathBuf::from(&mapping.path); + if !names_mapped_file(mapping, &path) { + continue; + } + + let load_bias = match ModuleSymbols::compute_load_bias( + &path, + mapping.avma_range.start, + mapping.avma_range.end, + mapping.file_offset, + ) { + Ok(load_bias) => load_bias, + Err(e) => { + debug!("Failed to compute load bias for {}: {e}", mapping.path); + continue; + } + }; + + let loaded_module = loaded_modules.entry(path.clone()).or_default(); + + if loaded_module.module_symbols.is_none() { + match ModuleSymbols::from_elf(&path) { + Ok(symbols) => loaded_module.module_symbols = Some(symbols), + Err(e) => debug!("Failed to load symbols for {}: {e}", mapping.path), + } + } + + // The ELF-derived halves are per file, the mounting is per mapping, so + // only the latter is recomputed for a module mapped more than once. + let unwind_data = match unwind_data_from_elf( + mapping.path.as_bytes(), + mapping.avma_range.start, + mapping.avma_range.end, + None, + load_bias, + ) { + Ok((unwind_data, mut process_unwind_data)) => { + process_unwind_data.timestamp = Some(mapping.timestamp); + Some((unwind_data, process_unwind_data)) + } + Err(e) => { + debug!("Failed to load unwind data for {}: {e}", mapping.path); + None + } + }; + + let process_loaded_module = loaded_module + .process_loaded_modules + .entry(mapping.pid) + .or_default(); + process_loaded_module.symbols_load_bias = Some(load_bias); + + if let Some((unwind_data, process_unwind_data)) = unwind_data { + loaded_module.unwind_data = Some(unwind_data); + process_loaded_module.process_unwind_data = Some(process_unwind_data); + } + } + + loaded_modules +} + +/// Whether the path still names the file that was mapped. +/// +/// The mapping records the inode the kernel resolved the path from; a file +/// rebuilt or replaced since then is a different inode, and reading unwind data +/// out of it would bind eh_frame from the wrong binary to those addresses. +fn names_mapped_file(mapping: &ProcessMapping, path: &Path) -> bool { + let Ok(metadata) = std::fs::metadata(path) else { + debug!("{} is no longer readable", mapping.path); + return false; + }; + + // The recorded `dev` is the kernel's s_dev encoding, `st_dev` glibc's, so + // only the decomposed major/minor pair is comparable. + let recorded = (mapping.dev >> 20, mapping.dev & 0xF_FFFF, mapping.ino); + let current = ( + u64::from(libc::major(metadata.dev())), + u64::from(libc::minor(metadata.dev())), + metadata.ino(), + ); + + if recorded != current { + debug!( + "{} changed since it was mapped (recorded {recorded:?}, now {current:?})", + mapping.path + ); + return false; + } + true +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + fn mapping_for(path: &str, dev: u64, ino: u64) -> ProcessMapping { + ProcessMapping { + pid: 42, + path: path.to_string(), + dev, + ino, + file_offset: 0, + avma_range: 0x1000..0x2000, + timestamp: 7, + } + } + + fn s_dev_of(path: &str) -> (u64, u64) { + let metadata = std::fs::metadata(path).unwrap(); + let dev = + u64::from(libc::major(metadata.dev())) << 20 | u64::from(libc::minor(metadata.dev())); + (dev, metadata.ino()) + } + + /// The recorded s_dev encoding and `st_dev` differ, so the check has to + /// decompose both or it rejects every module that did not change. + #[test] + fn accepts_a_file_that_still_has_the_recorded_inode() { + let path = "/proc/self/exe"; + let (dev, ino) = s_dev_of(path); + + assert!(names_mapped_file( + &mapping_for(path, dev, ino), + Path::new(path) + )); + } + + #[test] + fn rejects_a_file_whose_inode_changed() { + let path = "/proc/self/exe"; + let (dev, _) = s_dev_of(path); + + assert!(!names_mapped_file( + &mapping_for(path, dev, 0), + Path::new(path) + )); + } + + #[test] + fn rejects_a_path_that_no_longer_exists() { + let path = "/nonexistent/module.so"; + + assert!(!names_mapped_file( + &mapping_for(path, 1, 2), + Path::new(path) + )); + } + + /// The whole runner half of the pipeline: a recorded mapping in, keyed + /// unwind/symbol files plus a metadata referencing them out. + #[test] + fn writes_keyed_artifacts_and_metadata_for_a_recorded_mapping() { + const MODULE: &str = "testdata/perf_map/the_algorithms.bin"; + + let profile = tempfile::tempdir().unwrap(); + let results = profile.path().join("results"); + std::fs::create_dir_all(&results).unwrap(); + + let (dev, ino) = s_dev_of(MODULE); + MemtrackMappings { + mappings: vec![ProcessMapping { + pid: 1234, + path: MODULE.to_string(), + dev, + ino, + file_offset: 0x5_2000, + avma_range: 0x5555_555a_7000..0x5555_556b_0000, + timestamp: 999, + }], + } + .save_with_pid_to(&results, 1234) + .unwrap(); + + save_module_artifacts( + profile.path(), + &results, + ("codspeed-rust".to_string(), "4.2.0".to_string()), + ) + .unwrap(); + + let metadata = MemtrackMetadata::from_reader( + std::fs::File::open(profile.path().join("memtrack.metadata")).unwrap(), + ) + .unwrap(); + + assert_eq!(metadata.version, MEMTRACK_METADATA_CURRENT_VERSION); + assert_eq!( + metadata.artifacts.mapped_process_module_symbols[&1234].len(), + 1 + ); + + let unwind = &metadata.artifacts.mapped_process_unwind_data_by_pid[&1234][0]; + assert_eq!(unwind.inner.timestamp, Some(999)); + assert!( + profile + .path() + .join(format!("{}.unwind_data", unwind.unwind_data_key)) + .exists() + ); + assert_eq!( + metadata.artifacts.path_key_to_path[&unwind.unwind_data_key], + PathBuf::from(MODULE) + ); + } +} diff --git a/src/executor/shared/mod.rs b/src/executor/shared/mod.rs index 2badf4064..f278f07cd 100644 --- a/src/executor/shared/mod.rs +++ b/src/executor/shared/mod.rs @@ -1 +1,2 @@ pub mod fifo; +pub mod module_artifacts; diff --git a/src/executor/wall_time/profiler/perf/debug_info.rs b/src/executor/shared/module_artifacts/debug_info.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/debug_info.rs rename to src/executor/shared/module_artifacts/debug_info.rs diff --git a/src/executor/wall_time/profiler/perf/elf_helper.rs b/src/executor/shared/module_artifacts/elf_helper.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/elf_helper.rs rename to src/executor/shared/module_artifacts/elf_helper.rs diff --git a/src/executor/wall_time/profiler/perf/loaded_module.rs b/src/executor/shared/module_artifacts/loaded_module.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/loaded_module.rs rename to src/executor/shared/module_artifacts/loaded_module.rs diff --git a/src/executor/shared/module_artifacts/mod.rs b/src/executor/shared/module_artifacts/mod.rs new file mode 100644 index 000000000..72c2bf0ea --- /dev/null +++ b/src/executor/shared/module_artifacts/mod.rs @@ -0,0 +1,15 @@ +//! Extraction of symbols, unwind data and debug info from the ELF modules a +//! profiled process mapped, and their deduplicated on-disk layout. +//! +//! The input is a set of [`loaded_module::LoadedModule`]s, however the mappings +//! were discovered; the output is the keyed `unwind_data`/`symbols.map` files +//! plus the per-pid references that the metadata points at. + +mod elf_helper; +mod naming; + +pub mod debug_info; +pub mod loaded_module; +pub mod module_symbols; +pub mod save_artifacts; +pub mod unwind_data; diff --git a/src/executor/wall_time/profiler/perf/module_symbols.rs b/src/executor/shared/module_artifacts/module_symbols.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/module_symbols.rs rename to src/executor/shared/module_artifacts/module_symbols.rs diff --git a/src/executor/wall_time/profiler/perf/naming.rs b/src/executor/shared/module_artifacts/naming.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/naming.rs rename to src/executor/shared/module_artifacts/naming.rs diff --git a/src/executor/wall_time/profiler/perf/save_artifacts.rs b/src/executor/shared/module_artifacts/save_artifacts.rs similarity index 93% rename from src/executor/wall_time/profiler/perf/save_artifacts.rs rename to src/executor/shared/module_artifacts/save_artifacts.rs index 36b2fd12a..3e8903ed9 100644 --- a/src/executor/wall_time/profiler/perf/save_artifacts.rs +++ b/src/executor/shared/module_artifacts/save_artifacts.rs @@ -1,23 +1,22 @@ use super::debug_info::debug_info_by_path; use super::loaded_module::LoadedModule; +use super::naming; use crate::executor::valgrind::helpers::ignored_objects_path::get_objects_path_to_ignore; -use crate::executor::wall_time::profiler::perf::naming; use crate::prelude::*; use libc::pid_t; use rayon::prelude::*; use runner_shared::debug_info::{MappedProcessDebugInfo, ModuleDebugInfo}; +use runner_shared::metadata::ModuleArtifacts; use runner_shared::module_symbols::MappedProcessModuleSymbols; use runner_shared::unwind_data::{MappedProcessUnwindData, ProcessUnwindData, UnwindData}; use std::collections::HashMap; use std::path::{Path, PathBuf}; pub struct SavedArtifacts { - pub symbol_pid_mappings_by_pid: HashMap>, - pub debug_info: HashMap, - pub mapped_process_debug_info_by_pid: HashMap>, - pub mapped_process_unwind_data_by_pid: HashMap>, + pub artifacts: ModuleArtifacts, + /// Kept out of [`ModuleArtifacts`] because only the folded walltime trace + /// drops modules; other modes carry every module they mapped. pub ignored_modules_by_pid: HashMap>, - pub key_to_path: HashMap, } /// Save all artifacts (symbols, debug info, unwind data) from mounted modules and JIT data. @@ -30,7 +29,7 @@ pub fn save_artifacts( register_paths(&mut path_to_key, loaded_modules_by_path); - let symbol_pid_mappings_by_pid = + let mapped_process_module_symbols = save_symbols(profile_folder, loaded_modules_by_path, &path_to_key); let (debug_info, mapped_process_debug_info_by_pid) = @@ -45,18 +44,20 @@ pub fn save_artifacts( let ignored_modules_by_pid = collect_ignored_modules(loaded_modules_by_path); - let key_to_path = path_to_key + let path_key_to_path = path_to_key .into_iter() .map(|(path, key)| (key, path)) .collect(); SavedArtifacts { - symbol_pid_mappings_by_pid, - debug_info, - mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid, + artifacts: ModuleArtifacts { + debug_info, + mapped_process_debug_info_by_pid, + mapped_process_unwind_data_by_pid, + mapped_process_module_symbols, + path_key_to_path, + }, ignored_modules_by_pid, - key_to_path, } } diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap index 48d654070..9b917e545 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__cpp_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__cpp_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap index e92dcefa8..5b6a04ae2 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__golang_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__golang_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap index 75d0a4494..97dc7b43a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__ruff_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__ruff_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap index 6cf90c6a1..fd5e1aa0c 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__rust_divan_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__rust_divan_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap index 9e9c52a2e..a238cbb28 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__debug_info__tests__the_algorithms_debug_info.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__debug_info__tests__the_algorithms_debug_info.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/debug_info.rs +source: src/executor/shared/module_artifacts/debug_info.rs expression: module_debug_info.debug_infos --- [ diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap index 8456dd05b..34c79e04e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__cpp_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__cpp_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap index 84138e7bb..990e660d9 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__golang_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__golang_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap index 879d29f90..fe5907dd0 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__ruff_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__ruff_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap index 839039f35..10a77b716 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__rust_divan_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__rust_divan_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap similarity index 99% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap index fec3e2802..724f3002e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__module_symbols__tests__the_algorithms_symbols.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__module_symbols__tests__the_algorithms_symbols.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/perf/module_symbols.rs +source: src/executor/shared/module_artifacts/module_symbols.rs expression: module_symbols --- ModuleSymbols { diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap index 205b5e148..6c554024a 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__cpp_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__cpp_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap index 699e4b031..807e10611 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__golang_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__golang_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap index a0a5b0f98..3956fd7d1 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__ruff_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__ruff_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap index 0367c2dee..edfd8e558 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__rust_divan_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__rust_divan_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap similarity index 90% rename from src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap rename to src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap index 9fc15dca2..066c1ad0e 100644 --- a/src/executor/wall_time/profiler/perf/snapshots/codspeed_runner__executor__wall_time__profiler__perf__unwind_data__tests__the_algorithms_unwind_data.snap +++ b/src/executor/shared/module_artifacts/snapshots/codspeed_runner__executor__shared__module_artifacts__unwind_data__tests__the_algorithms_unwind_data.snap @@ -1,5 +1,5 @@ --- -source: src/executor/wall_time/profiler/perf/unwind_data.rs +source: src/executor/shared/module_artifacts/unwind_data.rs expression: "unwind_data_from_elf(module_path.as_bytes(), start_addr, end_addr, None,\nexpected_load_bias,)" --- Ok( diff --git a/src/executor/wall_time/profiler/perf/unwind_data.rs b/src/executor/shared/module_artifacts/unwind_data.rs similarity index 100% rename from src/executor/wall_time/profiler/perf/unwind_data.rs rename to src/executor/shared/module_artifacts/unwind_data.rs diff --git a/src/executor/wall_time/profiler/perf/jit_dump.rs b/src/executor/wall_time/profiler/perf/jit_dump.rs index fd5fad056..344f4080e 100644 --- a/src/executor/wall_time/profiler/perf/jit_dump.rs +++ b/src/executor/wall_time/profiler/perf/jit_dump.rs @@ -1,4 +1,4 @@ -use super::module_symbols::{ModuleSymbols, Symbol}; +use crate::executor::shared::module_artifacts::module_symbols::{ModuleSymbols, Symbol}; use crate::prelude::*; use linux_perf_data::jitdump::{JitDumpReader, JitDumpRecord}; use runner_shared::unwind_data::{ProcessUnwindData, UnwindData}; diff --git a/src/executor/wall_time/profiler/perf/mod.rs b/src/executor/wall_time/profiler/perf/mod.rs index 8816e5fb0..7f31921e2 100644 --- a/src/executor/wall_time/profiler/perf/mod.rs +++ b/src/executor/wall_time/profiler/perf/mod.rs @@ -10,6 +10,7 @@ use crate::executor::helpers::env::suppress_go_perf_unwinding_warning; use crate::executor::helpers::harvest_perf_maps_for_pids::harvest_perf_maps_for_pids; use crate::executor::helpers::run_with_sudo::wrap_with_sudo; use crate::executor::shared::fifo::FifoBenchmarkData; +use crate::executor::shared::module_artifacts::save_artifacts; use crate::executor::wall_time::profiler::NO_BENCHMARKS_DETECTED_WARNING; use crate::executor::wall_time::profiler::Profiler; use crate::executor::wall_time::profiler::SAMPLING_RATE_HZ; @@ -29,16 +30,9 @@ use runner_shared::metadata::WalltimeMetadata; use std::path::Path; use std::path::PathBuf; -mod debug_info; -mod elf_helper; mod jit_dump; -mod loaded_module; -mod module_symbols; -mod naming; mod parse_perf_file; -mod save_artifacts; pub(crate) mod setup; -mod unwind_data; pub mod fifo; pub mod perf_executable; @@ -306,11 +300,7 @@ impl BenchmarkData<'_> { uri_by_ts: self.marker_result.uri_by_ts.clone(), ignored_modules_by_pid: artifacts.ignored_modules_by_pid, markers: self.marker_result.markers.clone(), - debug_info: artifacts.debug_info, - mapped_process_debug_info_by_pid: artifacts.mapped_process_debug_info_by_pid, - mapped_process_unwind_data_by_pid: artifacts.mapped_process_unwind_data_by_pid, - mapped_process_module_symbols: artifacts.symbol_pid_mappings_by_pid, - path_key_to_path: artifacts.key_to_path, + artifacts: artifacts.artifacts, // Deprecated fields below are no longer used debug_info_by_pid: Default::default(), ignored_modules: Default::default(), diff --git a/src/executor/wall_time/profiler/perf/parse_perf_file.rs b/src/executor/wall_time/profiler/perf/parse_perf_file.rs index 151b54945..1d1033b38 100644 --- a/src/executor/wall_time/profiler/perf/parse_perf_file.rs +++ b/src/executor/wall_time/profiler/perf/parse_perf_file.rs @@ -1,6 +1,6 @@ -use super::loaded_module::{LoadedModule, ProcessLoadedModule}; -use super::module_symbols::ModuleSymbols; -use super::unwind_data::unwind_data_from_elf; +use crate::executor::shared::module_artifacts::loaded_module::{LoadedModule, ProcessLoadedModule}; +use crate::executor::shared::module_artifacts::module_symbols::ModuleSymbols; +use crate::executor::shared::module_artifacts::unwind_data::unwind_data_from_elf; use crate::prelude::*; use libc::pid_t; use linux_perf_data::PerfFileReader; diff --git a/src/executor/wall_time/profiler/samply/mod.rs b/src/executor/wall_time/profiler/samply/mod.rs index 3d77e7ade..5f04ef8c8 100644 --- a/src/executor/wall_time/profiler/samply/mod.rs +++ b/src/executor/wall_time/profiler/samply/mod.rs @@ -184,11 +184,7 @@ impl Profiler for SamplyProfiler { // These fields aren't required in samply, since we symbolicate client-side. ignored_modules_by_pid: Default::default(), - debug_info: Default::default(), - mapped_process_debug_info_by_pid: Default::default(), - mapped_process_unwind_data_by_pid: Default::default(), - mapped_process_module_symbols: Default::default(), - path_key_to_path: Default::default(), + artifacts: Default::default(), // Deprecated fields below are no longer used debug_info_by_pid: Default::default(),