From 97b0917138d9765ffa95b5b32279394806a5208f Mon Sep 17 00:00:00 2001 From: Borja Castellano Date: Mon, 14 Sep 2026 14:54:21 +0000 Subject: [PATCH] feat(dash-spv-bench): opt-in in-process CPU and heap profiling, client always in docker `./run.sh --flame` builds the bench with the `cpu-profile` feature (pprof, 99 Hz SIGPROF sampler) and writes flamegraph.svg. `--memory-snapshot` builds with `heap-profile` (jemalloc as the global allocator, sampling heap profiling) and writes heap-peak.svg: the live heap at the highest RSS of the run. A thread re-dumps the profile each time VmRSS grows by 5 % and only the last dump is rendered, after the sync. Without the flags neither profiler is compiled in. perf is not usable on the benchmark host (perf_event_paranoid=4) nor reliably under Docker Desktop, and jeprof needed path and placeholder workarounds, so both profilers now run inside the process. peak_rss_mib is read before the profiles are rendered: rendering the CPU flamegraph of a 20-minute run allocates ~2 GB, which otherwise showed up as a 4703 MiB peak and filled half of the heap snapshot. The client now always runs in the client container, shaped or not, and the binary is always built inside the Rust image, so it links against the container's glibc (a host build fails to start on hosts with a newer glibc than bookworm's). This drops the host-run path, the perf/perl/FlameGraph/OS checks and the policer fallback for kernels without ifb: a run whose download rate cannot be enforced now fails instead of producing a result that cannot be compared. `--wallets` with a relative path now resolves against the caller's directory instead of dash-spv-bench/. run.sh goes from 664 to 366 lines. Mainnet restore, mainnet.100mbi.100ms, on dev, --flame --memory-snapshot: 20.3 min, peak_rss_mib 2280, heap snapshot at 2243 MiB RSS (1.88 GB live), 14114383 sat, 13389 addresses. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017DruChNTWXwJoWPartZwCf --- dash-spv-bench/.gitignore | 4 - dash-spv-bench/Cargo.toml | 9 + dash-spv-bench/run.sh | 616 +++++++++------------------------- dash-spv-bench/src/main.rs | 15 +- dash-spv-bench/src/profile.rs | 91 +++++ 5 files changed, 271 insertions(+), 464 deletions(-) create mode 100644 dash-spv-bench/src/profile.rs diff --git a/dash-spv-bench/.gitignore b/dash-spv-bench/.gitignore index fb948c9f4..31fa05194 100644 --- a/dash-spv-bench/.gitignore +++ b/dash-spv-bench/.gitignore @@ -1,6 +1,5 @@ bench-results/ bench-storage/ -profiles/ *.lock @@ -17,8 +16,5 @@ chain-data/ results/ .bin/ -# FlameGraph tooling clone (run.sh --flame), fetched on demand. -.flamegraph/ - # Wallet mnemonics (one BIP39 phrase per line) — secrets, never committed. wallets.txt diff --git a/dash-spv-bench/Cargo.toml b/dash-spv-bench/Cargo.toml index 0cf11a8b9..58aa104b0 100644 --- a/dash-spv-bench/Cargo.toml +++ b/dash-spv-bench/Cargo.toml @@ -20,6 +20,15 @@ tracing = "0.1" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } indicatif = "0.18" +[target.'cfg(target_os = "linux")'.dependencies] +pprof = { version = "0.14", features = ["flamegraph"], optional = true } +tikv-jemallocator = { version = "0.7", features = ["profiling"], optional = true } +jemalloc_pprof = { version = "0.9", features = ["flamegraph", "symbolize"], optional = true } + +[features] +cpu-profile = ["dep:pprof"] +heap-profile = ["dep:tikv-jemallocator", "dep:jemalloc_pprof"] + [[bin]] name = "dash-spv-bench" path = "src/main.rs" diff --git a/dash-spv-bench/run.sh b/dash-spv-bench/run.sh index 09758e204..79e60be35 100755 --- a/dash-spv-bench/run.sh +++ b/dash-spv-bench/run.sh @@ -1,115 +1,68 @@ #!/bin/bash # -# dash-spv benchmark driver — runs ONE scenario end to end (build, bring up peers, sync, report). +# dash-spv benchmark driver: builds the client, brings up local peers when the +# scenario has them, syncs inside the client container and archives the results. # -# Usage: -# ./run.sh Run the scenario -# ./run.sh --flame Same, under the sampler (perf on Linux, sample on macOS) -# -> profiles/flamegraph.svg -# --wallets Wallets for this run: a file with one BIP39 mnemonic per -# line. No file => the run has no wallet. +# Usage: ./run.sh ... [--flame] [--memory-snapshot] [--wallets ] # -# ./run.sh 'scenarios/mainnet.*' Several scenarios: any argument that is not a file is -# ./run.sh scenarios/local.*.yml treated as a glob (quote it to let run.sh expand it, or -# ./run.sh scenarios/*.yml let your shell do it). Patterns resolve against the -# current directory and then against scenarios/, so -# 'mainnet.*' works from anywhere. +# --flame CPU flamegraph sampled by the client -> flamegraph.svg +# --memory-snapshot live-heap flamegraph at the RSS peak -> heap-peak.svg +# --wallets one BIP39 mnemonic per line (default: wallets.txt) # -# Every invocation, one scenario or twenty, writes results// containing per scenario -# .log stdout of the run (build, bring-up, summary) -# .run.log the sync trace, kept because bench-storage/ is wiped by the next run -# .summary.txt the metrics block -# plus results.tsv and report.md over all of them. A scenario that fails is recorded and the -# rest still run. -# -# RUST_LOG can be set to tweak logging, e.g. RUST_LOG=info ./run.sh scenarios/local.1ideal.yml -# It is the tracing filter for BOTH log sinks and overrides the defaults, which are -# terminal = "warn,dash_spv_bench=info" (kept light so the live bars stay readable) -# file = "warn,dash_spv=debug,dash_spv_bench=debug" (debug, for offline analysis) -# -# Outputs land in bench-storage/ (gitignored, wiped at the start of each run): -# run.log full trace of the sync (debug by default) for offline analysis -# summary.txt metrics + per-wallet tx/balance fingerprint (also printed to stdout) -# -# bench-storage/ is wiped at the start of every run, so both are archived into results/ too. -# plus the SPV storage the run produced (block_headers/, filters/, ...) +# An argument that is not a file is a glob, tried against the current directory +# and then scenarios/. Every invocation writes results// with, per +# scenario, .log, .run.log, .summary.txt and any flamegraph, plus +# results.tsv and report.md. RUST_LOG overrides the client's log filters. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -COMPOSE_FILE="" # local mode generates one here; deleted on exit -DASH_VERSION="23.1.7" -IMAGE="dash-spv-bench/dashd:${DASH_VERSION}" +SELF="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")" +INVOCATION_DIR="${PWD}" +IMAGE="dash-spv-bench/dashd:23.1.7" CLIENT_IMAGE="dash-spv-bench/client:1" -# Rust image for the Linux build, taken from the workspace's own pin so the two -# cannot drift: with a mismatched tag the image spends every run having rustup -# fetch the pinned toolchain before it can compile anything. RUST_CHANNEL="$(sed -n 's/^channel *= *"\(.*\)"/\1/p' "${REPO_ROOT}/rust-toolchain.toml" 2>/dev/null || true)" RUST_IMAGE="rust:${RUST_CHANNEL:-1.89}-bookworm" -# Separate from the host's `target/`: a different triple, and sharing one -# directory across both would make every switch a full rebuild. PROJECT="spv-bench" STATE="${SCRIPT_DIR}/.clonedir" -FLAME_SVG="${SCRIPT_DIR}/profiles/flamegraph.svg" -FLAMEGRAPH_DIR="${SCRIPT_DIR}/.flamegraph" # FlameGraph tooling clone (gitignored, inside the package) CHAIN_DIR="${SCRIPT_DIR}/chain-data" -# Absolute, because the script `cd`s to its own directory below: a relative `$0` -# stops resolving after that, which broke both `--help` and re-invoking self. -SELF="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")" - -INVOCATION_DIR="${PWD}" abspath() { case "$1" in /*) printf '%s\n' "$1" ;; *) printf '%s\n' "${INVOCATION_DIR%/}/$1" ;; esac; } - cd "${SCRIPT_DIR}" -FLAME=0 +FEATURES="" +FLAGS=() SCN_ARGS=() WALLETS_ARG="" while [ $# -gt 0 ]; do case "$1" in - --flame) FLAME=1; shift ;; - --wallets) WALLETS_ARG="${2:?--wallets needs a file path}"; shift 2 ;; - -h | --help) sed -n '3,24p' "${SELF}"; exit 0 ;; + --flame) FEATURES="${FEATURES} cpu-profile"; FLAGS+=("$1"); shift ;; + --memory-snapshot) FEATURES="${FEATURES} heap-profile"; FLAGS+=("$1"); shift ;; + --wallets) WALLETS_ARG="$(abspath "${2:?--wallets needs a file path}")"; FLAGS+=("$1" "${WALLETS_ARG}"); shift 2 ;; + -h | --help) sed -n '3,/^set /p' "${SELF}" | sed '$d'; exit 0 ;; -*) echo "unknown flag: $1" >&2; exit 1 ;; *) SCN_ARGS+=("$1"); shift ;; esac done -[ "${#SCN_ARGS[@]}" -gt 0 ] || { echo "usage: $0 ... [--flame] [--wallets ]" >&2; exit 1; } +[ "${#SCN_ARGS[@]}" -gt 0 ] || { echo "usage: $0 ... [--flame] [--memory-snapshot] [--wallets ]" >&2; exit 1; } -# Resolve each argument to scenario files. -# -# A bare path stays a bare path, so the original single-file invocation is -# untouched. Anything that is not a file is treated as a glob — which covers -# both a quoted pattern (`'scenarios/mainnet.*'`, expanded here) and one the -# caller's shell already expanded into several arguments. Patterns are tried -# against the invocation directory first and then against `scenarios/`, so -# `mainnet.*` works from anywhere. SCN_FILES=() +add_matches() { + local m + while IFS= read -r m; do + case "${m}" in /*) ;; *) m="$1/${m}" ;; esac + if [ -f "${m}" ]; then SCN_FILES+=("${m}"); fi + done < <(cd "$1" && compgen -G "$2") +} for arg in "${SCN_ARGS[@]}"; do - if [ -f "${arg}" ] || [ -f "$(abspath "${arg}")" ]; then - SCN_FILES+=("$(abspath "${arg}")") - continue - fi - matched=0 - while IFS= read -r hit; do - [ -f "${hit}" ] || continue - SCN_FILES+=("${hit}") - matched=1 - done < <(cd "${INVOCATION_DIR}" 2>/dev/null && shopt -s nullglob && printf '%s\n' ${arg} | while IFS= read -r m; do abspath "${m}"; done - cd "${SCRIPT_DIR}/scenarios" 2>/dev/null && shopt -s nullglob && printf '%s\n' ${arg} | while IFS= read -r m; do printf '%s\n' "${SCRIPT_DIR}/scenarios/${m}"; done) - [ "${matched}" -eq 1 ] || { echo "Error: no scenario matched: ${arg}" >&2; exit 1; } + n="${#SCN_FILES[@]}" + add_matches "${INVOCATION_DIR}" "${arg}" + [ "${#SCN_FILES[@]}" -gt "${n}" ] || add_matches "${SCRIPT_DIR}/scenarios" "${arg}" + [ "${#SCN_FILES[@]}" -gt "${n}" ] || { echo "Error: no scenario matched: ${arg}" >&2; exit 1; } done -# Run each scenario through a fresh invocation of this script and collect the -# numbers. Re-invoking rather than looping in place keeps the per-scenario path -# — exports, compose, traps, teardown — byte for byte what a single-file run has -# always done, so the extension cannot change the thing it is measuring. -# -# Taken for one scenario as much as for twenty: naming a file and naming a glob -# that happens to match one file are the same request, and having them leave -# their results in different shapes is a trap for anyone scripting on top. The -# child is marked so it runs the scenario instead of wrapping it again. +# Each scenario runs in a fresh invocation of this script, so one that fails +# cannot take the others with it. if [ -z "${BENCH_BATCH:-}" ]; then RUN_TS="$(date +%Y%m%d-%H%M%S)" OUT_DIR="${SCRIPT_DIR}/results/${RUN_TS}" @@ -117,11 +70,6 @@ if [ -z "${BENCH_BATCH:-}" ]; then TSV="${OUT_DIR}/results.tsv" REPORT="${OUT_DIR}/report.md" printf 'scenario\tcompleted\ttotal_ms\tblock_headers_ms\tfilter_headers_ms\tfilters_ms\ttransactions\tconfirmed_sat\tpeak_rss_mib\n' >"${TSV}" - - child_flags=() - [ "${FLAME}" -eq 1 ] && child_flags+=(--flame) - [ -n "${WALLETS_ARG}" ] && child_flags+=(--wallets "${WALLETS_ARG}") - metric() { awk -F':[[:space:]]*' -v k="$2" '$1==k {gsub(/[[:space:]]+$/,"",$2); print $2; exit}' "$1"; } echo "==> ${#SCN_FILES[@]} scenario(s); results in ${OUT_DIR}" @@ -129,37 +77,25 @@ if [ -z "${BENCH_BATCH:-}" ]; then name="$(basename "${f}" .yml)" log="${OUT_DIR}/${name}.log" echo "===== ${name} =====" - # A scenario that fails must not take the batch with it: record it and move - # on, or one bad run costs every result after it. BENCH_BATCH=1 BENCH_ARCHIVE_DIR="${OUT_DIR}" BENCH_ARCHIVE_NAME="${name}" \ - "${SELF}" "${f}" "${child_flags[@]+"${child_flags[@]}"}" 2>&1 | tee "${log}" || true - # Read the metrics from the summary the child archived, not from its stdout: - # the summary is written by the binary and is plain text either way, while a - # pty-captured log carries the bars' escape codes and carriage returns. - src="${log}" - if [ -s "${OUT_DIR}/${name}.summary.txt" ]; then src="${OUT_DIR}/${name}.summary.txt"; fi - wallet_line="$(grep -m1 -o 'confirmed_sat=[0-9]*' "${src}" || true)" - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ - "${name}" \ - "$(metric "${src}" completed)" \ - "$(metric "${src}" total_ms)" \ - "$(metric "${src}" block_headers_ms)" \ - "$(metric "${src}" filter_headers_ms)" \ - "$(metric "${src}" filters_ms)" \ - "$(metric "${src}" transactions)" \ - "$(sed -n 's/.*confirmed_sat=\([0-9]*\).*/\1/p' <<<"${wallet_line}")" \ - "$(metric "${src}" peak_rss_mib)" \ - >>"${TSV}" + "${SELF}" "${f}" ${FLAGS[@]+"${FLAGS[@]}"} 2>&1 | tee "${log}" || true + src="${OUT_DIR}/${name}.summary.txt" + [ -s "${src}" ] || src="${log}" + row="${name}" + for key in completed total_ms block_headers_ms filter_headers_ms filters_ms transactions; do + row+=$'\t'"$(metric "${src}" "${key}")" + done + row+=$'\t'"$(grep -m1 -o 'confirmed_sat=[0-9]*' "${src}" | cut -d= -f2 || true)" + row+=$'\t'"$(metric "${src}" peak_rss_mib)" + echo "${row}" >>"${TSV}" done { echo "# dash-spv bench — ${RUN_TS}" echo - echo "| scenario | completed | total_ms | headers_ms | filter_headers_ms | filters_ms | transactions | confirmed_sat | peak_rss_mib |" - echo "|---|---|---|---|---|---|---|---|---|" - tail -n +2 "${TSV}" | awk -F'\t' '{printf "| %s | %s | %s | %s | %s | %s | %s | %s | %s |\n", $1,$2,$3,$4,$5,$6,$7,$8,$9}' + awk -F'\t' '{ line = "|"; for (i = 1; i <= NF; i++) line = line " " $i " |"; print line + if (NR == 1) { line = "|"; for (i = 1; i <= NF; i++) line = line "---|"; print line } }' "${TSV}" } >"${REPORT}" - echo echo "==> report: ${REPORT}" cat "${REPORT}" @@ -167,33 +103,27 @@ if [ -z "${BENCH_BATCH:-}" ]; then fi SCN_FILE="${SCN_FILES[0]}" -[ -f "${SCN_FILE}" ] || { echo "Error: scenario file not found: ${SCN_FILE}" >&2; exit 1; } -YQ_VERSION="v4.44.6" -ensure_yq() { - if command -v yq >/dev/null 2>&1; then YQ=yq; return 0; fi - local bin="${SCRIPT_DIR}/.bin/yq" - if [ ! -x "${bin}" ]; then +if command -v yq >/dev/null 2>&1; then + YQ=yq +else + YQ="${SCRIPT_DIR}/.bin/yq" + if [ ! -x "${YQ}" ]; then + os="$(uname -s | tr '[:upper:]' '[:lower:]')" + case "$(uname -m)" in x86_64 | amd64) arch=amd64 ;; aarch64 | arm64) arch=arm64 ;; *) arch="$(uname -m)" ;; esac + echo "==> fetching yq into .bin/yq" mkdir -p "${SCRIPT_DIR}/.bin" - local os arch - os="$(uname -s | tr '[:upper:]' '[:lower:]')"; arch="$(uname -m)" - case "${arch}" in x86_64 | amd64) arch=amd64 ;; aarch64 | arm64) arch=arm64 ;; esac - echo "==> fetching yq ${YQ_VERSION} (${os}/${arch}) into .bin/yq" - curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_${os}_${arch}" \ - -o "${bin}" || { echo "Error: could not download yq; install it manually." >&2; exit 1; } - chmod +x "${bin}" + curl -fsSL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_${os}_${arch}" -o "${YQ}" \ + || { echo "Error: could not download yq; install it manually." >&2; exit 1; } + chmod +x "${YQ}" fi - YQ="${bin}" -} -ensure_yq -scn() { "${YQ}" "$1" "${SCN_FILE}"; } # evaluate a yq expression against the scenario file - -_peer_group() { - "${YQ}" ".peers[$1] | [.count, .latency_ms // 0, .jitter_ms // 0, .loss_pct // 0, .rate_kbit // 0, .corrupt_pct // 0, .reorder_pct // 0] | @tsv" "${SCN_FILE}" -} +fi +scn() { "${YQ}" "$1" "${SCN_FILE}"; } -build_netem() { - local lat="$1" jit="$2" loss="$3" rate="$4" corrupt="$5" reorder="$6" a="" +# `tc netem` arguments for the link described at yq path $1. +netem_args() { + local lat jit loss rate corrupt reorder a="" + read -r lat jit loss rate corrupt reorder <<<"$(scn "$1 | [.latency_ms // 0, .jitter_ms // 0, .loss_pct // 0, .rate_kbit // 0, .corrupt_pct // 0, .reorder_pct // 0] | @tsv")" [ "${lat}" != 0 ] && { a="delay ${lat}ms"; [ "${jit}" != 0 ] && a="${a} ${jit}ms"; } [ "${loss}" != 0 ] && a="${a} loss ${loss}%" [ "${rate}" != 0 ] && a="${a} rate ${rate}kbit" @@ -202,38 +132,8 @@ build_netem() { echo "${a# }" } -# The measured client's own link shaping, if the scenario asks for one -client_netem() { - local lat jit loss rate corrupt reorder - read -r lat jit loss rate corrupt reorder <<<"$("${YQ}" \ - '[.client.latency_ms // 0, .client.jitter_ms // 0, .client.loss_pct // 0, .client.rate_kbit // 0, .client.corrupt_pct // 0, .client.reorder_pct // 0] | @tsv' \ - "${SCN_FILE}")" - build_netem "${lat}" "${jit}" "${loss}" "${rate}" "${corrupt}" "${reorder}" -} - -# The client's bandwidth cap on its own, so the container can mirror it onto -# ingress -client_rate_kbit() { - "${YQ}" '.client.rate_kbit // 0' "${SCN_FILE}" -} - -peers_summary() { - local ng g count lat jit loss rate corrupt reorder tag out="" - ng="$(scn '.peers | length')" - case "${ng}" in ''|null|*[!0-9]*) ng=0 ;; esac - for ((g = 0; g < ng; g++)); do - read -r count lat jit loss rate corrupt reorder <<<"$(_peer_group "${g}")" - tag="${lat}ms" - [ "${jit}" != 0 ] && tag="${tag}±${jit}" - [ "${loss}" != 0 ] && tag="${tag}/${loss}%loss" - [ "${rate}" != 0 ] && tag="${tag}/${rate}kbit" - out="${out}, ${count}×${tag}" - done - echo "${out#, }" -} - emit_compose() { - local out="$1" peer_cpus="$2" + local out="$1" groups g count n peer=0 cat >"${out}" <
>"${out}" <>"${out}" </dev/null \ + if [ "\$\${INGRESS_RATE_KBIT:-0}" != 0 ]; then + ip link add ifb0 type ifb \ && ip link set ifb0 up \ && tc qdisc add dev eth0 handle ffff: ingress \ && tc filter add dev eth0 parent ffff: protocol all prio 1 u32 \ match u32 0 0 action mirred egress redirect dev ifb0 \ - && tc qdisc add dev ifb0 root netem rate \$\${INGRESS_RATE_KBIT}kbit; then - echo "client netem (ingress): rate \$\${INGRESS_RATE_KBIT}kbit via ifb0" - else - tc qdisc del dev eth0 ingress 2>/dev/null || true - if tc qdisc add dev eth0 handle ffff: ingress 2>/dev/null \ - && tc filter add dev eth0 parent ffff: protocol all prio 1 u32 \ - match u32 0 0 action police rate \$\${INGRESS_RATE_KBIT}kbit \ - burst \$\${INGRESS_BURST_K}k mtu 64k conform-exceed drop; then - echo "client police (ingress): rate \$\${INGRESS_RATE_KBIT}kbit burst \$\${INGRESS_BURST_K}k" - echo "NOTE: no ifb, so ingress is POLICED (drops) not shaped (queues) — throughput lands within ~3% but the loss pattern differs; do not compare against ifb runs" - else - echo "WARNING: ingress shaping failed (NET_ADMIN? no ifb and no act_police?)" - echo "WARNING: DOWNLOADS ARE UNSHAPED — rate_kbit is NOT enforced and this run is not comparable" - fi - fi + && tc qdisc add dev ifb0 root netem rate \$\${INGRESS_RATE_KBIT}kbit \ + || { echo "ERROR: cannot shape the client's downloads (no ifb in this kernel?)"; exit 1; } + echo "client netem (ingress): rate \$\${INGRESS_RATE_KBIT}kbit via ifb0" fi exec /usr/local/bin/dash-spv-bench SERVICE } - MODE="$(scn '.mode // "local"')" case "${MODE}" in local | testnet | mainnet) ;; *) echo "Error: mode must be 'local', 'testnet' or 'mainnet' (got '${MODE}')" >&2; exit 1 ;; esac -export BENCH_MODE="${MODE}" export CLONE_DIR="${CLONE_DIR:-/nonexistent}" -CLIENT_NETEM="$(client_netem)" -CLIENT_RATE_KBIT="$(client_rate_kbit)" +CLIENT_NETEM="$(netem_args .client)" +CLIENT_RATE_KBIT="$(scn '.client.rate_kbit // 0')" case "${CLIENT_RATE_KBIT}" in '' | *[!0-9]*) echo "Error: client.rate_kbit must be a whole number of kbit (got '${CLIENT_RATE_KBIT}')" >&2; exit 1 ;; esac -# Burst for the policer fallback: ~0.5s of the rate (kbit/16 => kbytes), floored -# so a very small rate still gets a workable bucket. Unused on the ifb path. -CLIENT_INGRESS_BURST_K=$(( CLIENT_RATE_KBIT / 16 )) -if [ "${CLIENT_INGRESS_BURST_K}" -lt 32 ]; then CLIENT_INGRESS_BURST_K=32; fi BENCH_CPUS="$(scn '.cpus // ""')" -BENCH_MAX_PEERS="$(scn '.max_peers // ""')" # only if set; else the ClientConfig default +BENCH_MAX_PEERS="$(scn '.max_peers // ""')" export BENCH_MAX_PEERS - -BENCH_WALLET_FILE="${SCRIPT_DIR}/wallets.txt" -[ -n "${WALLETS_ARG}" ] && BENCH_WALLET_FILE="$(abspath "${WALLETS_ARG}")" -export BENCH_WALLET_FILE +BENCH_WALLET_FILE="${WALLETS_ARG:-${SCRIPT_DIR}/wallets.txt}" export BENCH_STORAGE_DIR="${SCRIPT_DIR}/bench-storage" DESC="$(scn '.description // ""')" -[ -n "${DESC}" ] && echo "==> description: ${DESC}" +[ -z "${DESC}" ] || echo "==> description: ${DESC}" -BLOCKS="" if [ "${MODE}" = local ]; then - BLOCKS="$(scn '.blocks // 1000000')" - export BENCH_HEIGHT="${BLOCKS}" - unset BENCH_START_HEIGHT # local always syncs from genesis + BENCH_HEIGHT="$(scn '.blocks // 1000000')" + export BENCH_HEIGHT + unset BENCH_START_HEIGHT else BENCH_PEERS="$(scn '.peers // [] | join(",")')" export BENCH_PEERS - sh="$(scn '.start_height // ""')"; [ -n "${sh}" ] && export BENCH_START_HEIGHT="${sh}" + start_height="$(scn '.start_height // ""')" + [ -z "${start_height}" ] || export BENCH_START_HEIGHT="${start_height}" fi -expand_cpus() { - local part lo hi - local IFS=, - for part in $1; do - case "${part}" in - *-*) lo="${part%-*}"; hi="${part#*-}"; seq "${lo}" "${hi}" ;; - *) echo "${part}" ;; - esac - done -} - -CPU_PREFIX=() +# The client gets BENCH_CPUS, docker peers get every other core. +BENCH_PEER_CPUS="" if [ -n "${BENCH_CPUS}" ]; then ncpu="$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 0)" - bench_cores="$(expand_cpus "${BENCH_CPUS}" | sort -nu)" - if [ "${ncpu}" -gt 0 ]; then - peer_cores="" - for i in $(seq 0 $((ncpu - 1))); do - grep -qxF "${i}" <<<"${bench_cores}" || peer_cores="${peer_cores},${i}" - done - BENCH_PEER_CPUS="${peer_cores#,}" - fi - - if [ -n "${CLIENT_NETEM}" ]; then - # The client is a container: `cpuset` on the service does the pinning that - # `taskset` does for a host run, and it works on macOS where taskset does - # not exist at all. Emitted by `emit_client_service` from BENCH_CPUS. - echo "==> pinning the measured client container to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" - elif command -v taskset >/dev/null 2>&1; then - CPU_PREFIX=(taskset -c "${BENCH_CPUS}") - echo "==> pinning the measured run to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" - else - echo "==> note: 'taskset' not found (e.g. macOS); running the measured binary UNPINNED${BENCH_PEER_CPUS:+ (docker peers still pinned to ${BENCH_PEER_CPUS})}" >&2 - fi -fi - -# Where the measured binary will be, decided before the compose that mounts -# it is written. Everything lands in the workspace `target/`: a cross build -# gets cargo's own per-triple subdirectory, so it cannot collide with a host -# build, and neither duplicates the dependency graph. -if [ -n "${CLIENT_NETEM}" ] && [ "$(uname -s)" != Linux ]; then - CROSS_TRIPLE="$(docker run --rm "${RUST_IMAGE}" rustc -vV | sed -n 's/^host: //p')" - [ -n "${CROSS_TRIPLE}" ] || { echo "could not read the build image's target triple" >&2; exit 1; } - BIN="${REPO_ROOT}/target/${CROSS_TRIPLE}/release/dash-spv-bench" -else - CROSS_TRIPLE="" - BIN="${REPO_ROOT}/target/release/dash-spv-bench" -fi - -# A compose file is needed for the peers (local mode) and for the client -# container (any mode with a `client:` section) — testnet/mainnet shape the -# client against the real network, with no peer services at all. -if [ "${MODE}" = local ] || [ -n "${CLIENT_NETEM}" ]; then - COMPOSE_FILE="$(mktemp "${SCRIPT_DIR}/.scenario.XXXXXX")" - mv "${COMPOSE_FILE}" "${COMPOSE_FILE}.yml" - COMPOSE_FILE="${COMPOSE_FILE}.yml" - npeers="$(emit_compose "${COMPOSE_FILE}" "${BENCH_PEER_CPUS:-}")" - if [ "${MODE}" = local ]; then - echo "==> scenario '$(basename "${SCN_FILE}" .yml)': ${npeers} peers [$(peers_summary)], cpus=${BENCH_CPUS:-}, blocks=${BLOCKS}" - fi + bench_cores="$(IFS=,; for part in ${BENCH_CPUS}; do case "${part}" in *-*) seq "${part%-*}" "${part#*-}" ;; *) echo "${part}" ;; esac; done)" + for ((i = 0; i < ncpu; i++)); do + grep -qxF "${i}" <<<"${bench_cores}" || BENCH_PEER_CPUS="${BENCH_PEER_CPUS},${i}" + done + BENCH_PEER_CPUS="${BENCH_PEER_CPUS#,}" + echo "==> pinning the client container to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" fi -[ -n "${CLIENT_NETEM}" ] && echo "==> client link shaped: ${CLIENT_NETEM}" +# Built in the Rust image so the binary links against the client image's glibc. +BIN="${REPO_ROOT}/target/bench/release/dash-spv-bench" +echo "==> building bench binary in ${RUST_IMAGE}" +docker run --rm -v "${REPO_ROOT}:/src" -w /src --user "$(id -u):$(id -g)" \ + -e CARGO_HOME=/src/target/bench/cargo-home -e CARGO_TARGET_DIR=/src/target/bench \ + -e CARGO_NET_GIT_FETCH_WITH_CLI=true -e CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ + "${RUST_IMAGE}" cargo build --release -p dash-spv-bench ${FEATURES:+--features "${FEATURES# }"} +[ -x "${BIN}" ] || { echo "build produced no binary at ${BIN}" >&2; exit 1; } + +COMPOSE_FILE="$(mktemp "${SCRIPT_DIR}/.scenario.XXXXXX")" +mv "${COMPOSE_FILE}" "${COMPOSE_FILE}.yml" +COMPOSE_FILE="${COMPOSE_FILE}.yml" +emit_compose "${COMPOSE_FILE}" +[ -z "${CLIENT_NETEM}" ] || echo "==> client link shaped: ${CLIENT_NETEM}" compose() { docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@"; } +teardown() { + compose down --remove-orphans >/dev/null 2>&1 || true + [ ! -f "${STATE}" ] || rm -rf "$(cat "${STATE}")" "${STATE}" 2>/dev/null || true + rm -rf "${SCRIPT_DIR}"/.bench-clones.* 2>/dev/null || true +} +trap 'teardown; rm -f "${COMPOSE_FILE}"' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +trap 'exit 129' HUP + wait_loaded() { - local c logs + local c logs all for _ in $(seq 1 400); do - local all=1 + all=1 for c in "$@"; do logs="$(docker logs "${c}" 2>&1)" || { all=0; break; } case "${logs}" in *"init message: Done loading"*) ;; *) all=0; break ;; esac done - [ "${all}" -eq 1 ] && return 0 + [ "${all}" -eq 0 ] || return 0 echo -n "."; sleep 3 done return 1 } -teardown() { - [ -n "${COMPOSE_FILE}" ] && compose down --remove-orphans >/dev/null 2>&1 || true - [ -f "${STATE}" ] && { rm -rf "$(cat "${STATE}")" 2>/dev/null || true; rm -f "${STATE}"; } - rm -rf "${SCRIPT_DIR}"/.bench-clones.* 2>/dev/null || true -} - -arm_teardown() { - # On exit: tear down, THEN delete the generated compose (down needs it to still exist). - trap 'teardown; [ -n "${COMPOSE_FILE}" ] && rm -f "${COMPOSE_FILE}"' EXIT - trap 'exit 130' INT - trap 'exit 143' TERM - trap 'exit 129' HUP -} - -bring_up() { +if [ "${MODE}" = local ]; then + bash "${SCRIPT_DIR}/snapshot-chain.sh" # builds ./chain-data to BENCH_HEIGHT via docker echo "==> ensuring a clean network" teardown docker image inspect "${IMAGE}" >/dev/null 2>&1 || { echo "==> building peer image"; compose build; } - - # The client is in the same compose file but is not a peer: it must not be - # counted, started here, or waited on for "Done loading". - local services; services="$(compose config --services | grep -v '^client$' || true)" - local n; n="$(echo ${services} | wc -w | tr -d ' ')" - # BENCH_PEERS is derived from the generated peers (host ports 19401..). - local peers_csv="" - for svc in ${services}; do peers_csv="${peers_csv},127.0.0.1:$((19400 + ${svc#dashd}))"; done - export BENCH_PEERS="${peers_csv#,}" - - echo "==> CoW-cloning ${CHAIN_DIR} for ${n} peers (instant)" - local clone_dir; clone_dir="$(mktemp -d "${SCRIPT_DIR}/.bench-clones.XXXXXX")" - echo "${clone_dir}" > "${STATE}" + services="$(compose config --services | grep -v '^client$' || true)" + npeers="$(echo ${services} | wc -w | tr -d ' ')" + peers_summary="$(scn '[.peers[] | (.count | tostring) + "×" + ([to_entries[] | select(.key != "count") | .key + "=" + (.value | tostring)] | join(" "))] | join(", ")')" + echo "==> scenario '$(basename "${SCN_FILE}" .yml)': ${npeers} peers [${peers_summary}], cpus=${BENCH_CPUS:-}, blocks=${BENCH_HEIGHT}" + echo "==> CoW-cloning ${CHAIN_DIR} for ${npeers} peers" + clone_dir="$(mktemp -d "${SCRIPT_DIR}/.bench-clones.XXXXXX")" + echo "${clone_dir}" >"${STATE}" for svc in ${services}; do - local dst="${clone_dir}/peer${svc#dashd}" + dst="${clone_dir}/peer${svc#dashd}" cp -c -R "${CHAIN_DIR}" "${dst}" 2>/dev/null \ || cp --reflink=auto -R "${CHAIN_DIR}" "${dst}" 2>/dev/null \ || cp -R "${CHAIN_DIR}" "${dst}" done - - local batch=4 - echo "==> starting ${n} peers in batches of ${batch}" - local started="" count=0 + echo "==> starting ${npeers} peers in batches of 4" + started="" + count=0 for svc in ${services}; do CLONE_DIR="${clone_dir}" compose up -d "${svc}" >/dev/null 2>&1 - started="${started} spv-bench-${svc}"; count=$((count + 1)) - if [ $((count % batch)) -eq 0 ]; then - echo -n " loaded ${count}/${n} " + started="${started} spv-bench-${svc}" + count=$((count + 1)) + if [ $((count % 4)) -eq 0 ] || [ "${count}" -eq "${npeers}" ]; then + echo -n " loaded ${count}/${npeers} " wait_loaded ${started} || { echo " timeout loading batch" >&2; exit 1; } echo " ok" fi done - if [ $((count % batch)) -ne 0 ]; then # wait on the trailing partial batch too - echo -n " loaded ${count}/${n} " - wait_loaded ${started} || { echo " timeout loading batch" >&2; exit 1; } - echo " ok" - fi - - echo "==> ${n} peers started" - - # A containerised client shares the compose network with the peers, so it must - # reach them at their container addresses; the published host ports only exist - # for a client running on the host. Read the addresses back from the running - # containers rather than pinning a subnet in the compose file: a pinned subnet - # is one more thing that can collide with whatever else the machine has up, - # VPNs included, and it buys nothing a lookup does not. - if [ -n "${CLIENT_NETEM}" ]; then - local ip csv="" - for svc in ${services}; do - ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "spv-bench-${svc}")" - [ -n "${ip}" ] || { echo "Error: could not resolve address of spv-bench-${svc}" >&2; exit 1; } - csv="${csv},${ip}:19400" - done - export BENCH_PEERS="${csv#,}" - echo "==> client will reach peers at ${BENCH_PEERS}" - fi -} - -build_bin() { - if [ -n "${CROSS_TRIPLE}" ]; then - # The client runs in a Linux container, so the binary has to be a Linux - # one. On a Linux host the ordinary build already is — same triple, and - # glibc is forward compatible — so only a non-Linux host comes here. - # Bind-mounts the workspace and builds into its `target/`, which cargo - # keeps under the triple, so this stays incremental — an image layer build - # would replay the whole workspace on every source change, which makes A/B - # runs unusable. - echo "==> cross-building bench binary (${RUST_IMAGE}, ${CROSS_TRIPLE})" - # As the invoking user, with CARGO_HOME inside the workspace: writing - # into the shared `target/` as root leaves artifacts the host build then - # cannot overwrite. - docker run --rm \ - -v "${REPO_ROOT}:/src" \ - -w /src \ - --user "$(id -u):$(id -g)" \ - -e CARGO_HOME=/src/target/.cross-cargo-home \ - -e CARGO_TARGET_DIR=/src/target \ - -e CARGO_NET_GIT_FETCH_WITH_CLI=true \ - -e CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ - "${RUST_IMAGE}" \ - cargo build --release --target "${CROSS_TRIPLE}" -p dash-spv-bench - else - echo "==> building bench binary (release + line-table symbols)" - ( cd "${REPO_ROOT}" && CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ - cargo build --release -p dash-spv-bench ) - fi - [ -x "${BIN}" ] || { echo "build produced no binary at ${BIN}" >&2; exit 1; } -} - -FLAME_TOOL="" -if [ "${FLAME}" -eq 1 ]; then # fail fast on missing profiler deps, before building - if command -v perf >/dev/null; then FLAME_TOOL=perf # Linux - elif command -v /usr/bin/sample >/dev/null; then FLAME_TOOL=sample # macOS - else echo "flame mode needs 'perf' (Linux) or '/usr/bin/sample' (macOS)" >&2; exit 1; fi - [ -f "${FLAMEGRAPH_DIR}/flamegraph.pl" ] || \ - git clone --depth 1 https://github.com/brendangregg/FlameGraph "${FLAMEGRAPH_DIR}" -fi - -build_bin - -if [ "${MODE}" = local ]; then - arm_teardown - bash "${SCRIPT_DIR}/snapshot-chain.sh" # builds ./chain-data to BENCH_HEIGHT via docker - bring_up + # Container addresses, read back rather than pinned to a subnet that could + # collide with whatever else the machine has up. + csv="" + for svc in ${services}; do + ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "spv-bench-${svc}")" + [ -n "${ip}" ] || { echo "Error: could not resolve address of spv-bench-${svc}" >&2; exit 1; } + csv="${csv},${ip}:19400" + done + export BENCH_PEERS="${csv#,}" + echo "==> ${npeers} peers started, reachable at ${BENCH_PEERS}" else echo "==> ${MODE} mode, peers: ${BENCH_PEERS:-}" fi -# Run the sync with its live output straight on the terminal instead of through -# the batch wrapper's pipe -run_live() { - # Not `[ -w /dev/tty ]`: the device node is writable even with no - # controlling terminal, and the redirect then fails with ENXIO. Only - # actually opening it answers the question. - if { : >/dev/tty; } 2>/dev/null; then - "$@" >/dev/tty 2>&1 - else - "$@" - fi -} - -if [ -n "${CLIENT_NETEM}" ]; then - # `run` rather than `up`: it streams the client's own stdout and gives back - # its exit code, which is what the report is read from. Profiling is not - # wired through the container, so `--flame` is refused rather than silently - # producing a graph of the host doing nothing. - [ "${FLAME}" -eq 0 ] || { echo "Error: --flame is not supported with a containerised client" >&2; exit 1; } - arm_teardown - echo "==> running sync in the client container" - run_live compose run --rm --build client -elif [ "${FLAME}" -eq 0 ]; then - echo "==> running sync" - run_live "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" -elif [ "${FLAME_TOOL}" = perf ]; then - echo "==> running sync under perf" - mkdir -p "$(dirname "${FLAME_SVG}")" - perf record -F 499 -g -o /tmp/bench-perf.data -- "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" - perf script -i /tmp/bench-perf.data \ - | "${FLAMEGRAPH_DIR}/stackcollapse-perf.pl" \ - | "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "dash-spv sync" --colors hot > "${FLAME_SVG}" - echo "==> wrote ${FLAME_SVG}" +echo "==> running sync in the client container" +if { : >/dev/tty; } 2>/dev/null; then + compose run --rm --build client >/dev/tty 2>&1 else - echo "==> running sync under the sampler" - "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" & bpid=$! - sleep 3 - /usr/bin/sample "${bpid}" 2000 1 -file /tmp/bench.sample.txt -mayDie >/dev/null 2>&1 || true - wait "${bpid}" 2>/dev/null || true - mkdir -p "$(dirname "${FLAME_SVG}")" - "${FLAMEGRAPH_DIR}/stackcollapse-sample.awk" /tmp/bench.sample.txt \ - | sed -E 's/^Thread_[^;]*;//' \ - | "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "dash-spv sync" --colors hot > "${FLAME_SVG}" - echo "==> wrote ${FLAME_SVG}" + compose run --rm --build client fi -# Keep this run's outputs, which `bench-storage/` does not: it is wiped at the -# start of every run, so without this the trace of anything but the most recent -# scenario is gone — and in a batch that meant 15 of 16 syncs left nothing to -# look at afterwards. -# -# The batch passes its own directory in, so a child archives straight into it -# and there is one copy, named after the scenario. A lone run makes its own. -ARCHIVE_DIR="${BENCH_ARCHIVE_DIR:-${SCRIPT_DIR}/results/$(date +%Y%m%d-%H%M%S)-$(basename "${SCN_FILE}" .yml)}" -# Prefixed with the scenario only inside a batch, where one directory holds -# every scenario. A lone run's directory is already named after it. -ARCHIVE_NAME="${BENCH_ARCHIVE_NAME:-}" +ARCHIVE_DIR="${BENCH_ARCHIVE_DIR:?BENCH_ARCHIVE_DIR is set by the batch wrapper}" mkdir -p "${ARCHIVE_DIR}" -for out in run.log summary.txt; do +for out in run.log summary.txt flamegraph.svg heap-peak.svg; do [ -s "${BENCH_STORAGE_DIR}/${out}" ] || continue - cp "${BENCH_STORAGE_DIR}/${out}" "${ARCHIVE_DIR}/${ARCHIVE_NAME:+${ARCHIVE_NAME}.}${out}" + cp "${BENCH_STORAGE_DIR}/${out}" "${ARCHIVE_DIR}/${BENCH_ARCHIVE_NAME}.${out}" done echo "==> logs archived to ${ARCHIVE_DIR}" diff --git a/dash-spv-bench/src/main.rs b/dash-spv-bench/src/main.rs index 03b34da7c..35544319b 100644 --- a/dash-spv-bench/src/main.rs +++ b/dash-spv-bench/src/main.rs @@ -1,5 +1,7 @@ mod dashboard; mod metrics; +#[cfg(target_os = "linux")] +mod profile; use std::net::SocketAddr; use std::path::PathBuf; @@ -40,11 +42,11 @@ fn load_mnemonics() -> Vec { .unwrap_or_default() } -fn peak_rss_kb() -> Option { +fn proc_status_kb(field: &str) -> Option { std::fs::read_to_string("/proc/self/status") .ok()? .lines() - .find_map(|line| line.strip_prefix("VmHWM:"))? + .find_map(|line| line.strip_prefix(field))? .split_whitespace() .next()? .parse() @@ -89,6 +91,9 @@ async fn main() -> Result<()> { ) .init(); + #[cfg(target_os = "linux")] + let profilers = profile::start(); + let mode = env_or("BENCH_MODE", "local").trim().to_ascii_lowercase(); let (network, remote) = match mode.as_str() { "local" => (Network::Testnet, false), @@ -197,9 +202,13 @@ async fn main() -> Result<()> { run_handle.abort(); let _ = run_handle.await; + let peak_rss_kb = proc_status_kb("VmHWM:"); + #[cfg(target_os = "linux")] + profilers.finish(&output_dir); + use std::fmt::Write as _; let mut report = format!("{m}\n"); - if let Some(kb) = peak_rss_kb() { + if let Some(kb) = peak_rss_kb { let _ = writeln!(report, "peak_rss_mib: {}", kb / 1024); } diff --git a/dash-spv-bench/src/profile.rs b/dash-spv-bench/src/profile.rs new file mode 100644 index 000000000..05735115a --- /dev/null +++ b/dash-spv-bench/src/profile.rs @@ -0,0 +1,91 @@ +use std::path::Path; + +#[cfg(feature = "heap-profile")] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[cfg(feature = "heap-profile")] +#[export_name = "_rjem_malloc_conf"] +pub static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; + +#[cfg(feature = "heap-profile")] +type HeapPeak = std::sync::Arc>>; + +pub(crate) struct Profilers { + #[cfg(feature = "cpu-profile")] + cpu: Option>, + #[cfg(feature = "heap-profile")] + heap_peak: HeapPeak, +} + +pub(crate) fn start() -> Profilers { + let profilers = Profilers { + #[cfg(feature = "cpu-profile")] + cpu: pprof::ProfilerGuardBuilder::default() + .frequency(99) + .blocklist(&["libc", "libgcc", "pthread", "vdso"]) + .build() + .inspect_err(|e| tracing::warn!("cpu profiler not started: {e}")) + .ok(), + #[cfg(feature = "heap-profile")] + heap_peak: HeapPeak::default(), + }; + #[cfg(feature = "heap-profile")] + { + let peak = profilers.heap_peak.clone(); + std::thread::spawn(move || { + let Some(ctl) = jemalloc_pprof::PROF_CTL.as_ref() else { + tracing::warn!("jemalloc heap profiling is not enabled"); + return; + }; + let mut dumped_kb = 0; + while let Some(rss_kb) = crate::proc_status_kb("VmRSS:") { + if rss_kb > dumped_kb + dumped_kb / 20 { + match ctl.blocking_lock().dump_profile() { + Ok(profile) => { + dumped_kb = rss_kb; + if let Ok(mut slot) = peak.lock() { + *slot = Some((rss_kb, profile)); + } + } + Err(e) => { + tracing::warn!("heap snapshot failed: {e}"); + return; + } + } + } + std::thread::sleep(std::time::Duration::from_millis(500)); + } + }); + } + profilers +} + +impl Profilers { + #[allow(unused_variables)] + pub(crate) fn finish(self, dir: &Path) { + #[cfg(feature = "heap-profile")] + if let Some((rss_kb, profile)) = self.heap_peak.lock().ok().and_then(|mut slot| slot.take()) + { + let mut opts = jemalloc_pprof::FlamegraphOptions::default(); + opts.title = format!("dash-spv live heap at the RSS peak ({} MiB)", rss_kb / 1024); + opts.count_name = "bytes".to_string(); + let written = + profile.to_flamegraph(&mut opts).map_err(|e| e.to_string()).and_then(|svg| { + std::fs::write(dir.join("heap-peak.svg"), svg).map_err(|e| e.to_string()) + }); + if let Err(e) = written { + tracing::warn!("could not write heap-peak.svg: {e}"); + } + } + #[cfg(feature = "cpu-profile")] + if let Some(report) = self.cpu.as_ref().and_then(|guard| guard.report().build().ok()) { + let written = std::fs::File::create(dir.join("flamegraph.svg")) + .map_err(|e| e.to_string()) + .and_then(|file| report.flamegraph(file).map_err(|e| e.to_string())); + if let Err(e) = written { + tracing::warn!("could not write flamegraph.svg: {e}"); + } + } + } +}