diff --git a/dash-spv-bench/.gitignore b/dash-spv-bench/.gitignore index fb948c9f4..31fa05194 100644 --- a/dash-spv-bench/.gitignore +++ b/dash-spv-bench/.gitignore @@ -1,6 +1,5 @@ bench-results/ bench-storage/ -profiles/ *.lock @@ -17,8 +16,5 @@ chain-data/ results/ .bin/ -# FlameGraph tooling clone (run.sh --flame), fetched on demand. -.flamegraph/ - # Wallet mnemonics (one BIP39 phrase per line) — secrets, never committed. wallets.txt diff --git a/dash-spv-bench/Cargo.toml b/dash-spv-bench/Cargo.toml index 0cf11a8b9..58aa104b0 100644 --- a/dash-spv-bench/Cargo.toml +++ b/dash-spv-bench/Cargo.toml @@ -20,6 +20,15 @@ tracing = "0.1" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } indicatif = "0.18" +[target.'cfg(target_os = "linux")'.dependencies] +pprof = { version = "0.14", features = ["flamegraph"], optional = true } +tikv-jemallocator = { version = "0.7", features = ["profiling"], optional = true } +jemalloc_pprof = { version = "0.9", features = ["flamegraph", "symbolize"], optional = true } + +[features] +cpu-profile = ["dep:pprof"] +heap-profile = ["dep:tikv-jemallocator", "dep:jemalloc_pprof"] + [[bin]] name = "dash-spv-bench" path = "src/main.rs" diff --git a/dash-spv-bench/run.sh b/dash-spv-bench/run.sh index 09758e204..79e60be35 100755 --- a/dash-spv-bench/run.sh +++ b/dash-spv-bench/run.sh @@ -1,115 +1,68 @@ #!/bin/bash # -# dash-spv benchmark driver — runs ONE scenario end to end (build, bring up peers, sync, report). +# dash-spv benchmark driver: builds the client, brings up local peers when the +# scenario has them, syncs inside the client container and archives the results. # -# Usage: -# ./run.sh Run the scenario -# ./run.sh --flame Same, under the sampler (perf on Linux, sample on macOS) -# -> profiles/flamegraph.svg -# --wallets Wallets for this run: a file with one BIP39 mnemonic per -# line. No file => the run has no wallet. +# Usage: ./run.sh ... [--flame] [--memory-snapshot] [--wallets ] # -# ./run.sh 'scenarios/mainnet.*' Several scenarios: any argument that is not a file is -# ./run.sh scenarios/local.*.yml treated as a glob (quote it to let run.sh expand it, or -# ./run.sh scenarios/*.yml let your shell do it). Patterns resolve against the -# current directory and then against scenarios/, so -# 'mainnet.*' works from anywhere. +# --flame CPU flamegraph sampled by the client -> flamegraph.svg +# --memory-snapshot live-heap flamegraph at the RSS peak -> heap-peak.svg +# --wallets one BIP39 mnemonic per line (default: wallets.txt) # -# Every invocation, one scenario or twenty, writes results// containing per scenario -# .log stdout of the run (build, bring-up, summary) -# .run.log the sync trace, kept because bench-storage/ is wiped by the next run -# .summary.txt the metrics block -# plus results.tsv and report.md over all of them. A scenario that fails is recorded and the -# rest still run. -# -# RUST_LOG can be set to tweak logging, e.g. RUST_LOG=info ./run.sh scenarios/local.1ideal.yml -# It is the tracing filter for BOTH log sinks and overrides the defaults, which are -# terminal = "warn,dash_spv_bench=info" (kept light so the live bars stay readable) -# file = "warn,dash_spv=debug,dash_spv_bench=debug" (debug, for offline analysis) -# -# Outputs land in bench-storage/ (gitignored, wiped at the start of each run): -# run.log full trace of the sync (debug by default) for offline analysis -# summary.txt metrics + per-wallet tx/balance fingerprint (also printed to stdout) -# -# bench-storage/ is wiped at the start of every run, so both are archived into results/ too. -# plus the SPV storage the run produced (block_headers/, filters/, ...) +# An argument that is not a file is a glob, tried against the current directory +# and then scenarios/. Every invocation writes results// with, per +# scenario, .log, .run.log, .summary.txt and any flamegraph, plus +# results.tsv and report.md. RUST_LOG overrides the client's log filters. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" -COMPOSE_FILE="" # local mode generates one here; deleted on exit -DASH_VERSION="23.1.7" -IMAGE="dash-spv-bench/dashd:${DASH_VERSION}" +SELF="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")" +INVOCATION_DIR="${PWD}" +IMAGE="dash-spv-bench/dashd:23.1.7" CLIENT_IMAGE="dash-spv-bench/client:1" -# Rust image for the Linux build, taken from the workspace's own pin so the two -# cannot drift: with a mismatched tag the image spends every run having rustup -# fetch the pinned toolchain before it can compile anything. RUST_CHANNEL="$(sed -n 's/^channel *= *"\(.*\)"/\1/p' "${REPO_ROOT}/rust-toolchain.toml" 2>/dev/null || true)" RUST_IMAGE="rust:${RUST_CHANNEL:-1.89}-bookworm" -# Separate from the host's `target/`: a different triple, and sharing one -# directory across both would make every switch a full rebuild. PROJECT="spv-bench" STATE="${SCRIPT_DIR}/.clonedir" -FLAME_SVG="${SCRIPT_DIR}/profiles/flamegraph.svg" -FLAMEGRAPH_DIR="${SCRIPT_DIR}/.flamegraph" # FlameGraph tooling clone (gitignored, inside the package) CHAIN_DIR="${SCRIPT_DIR}/chain-data" -# Absolute, because the script `cd`s to its own directory below: a relative `$0` -# stops resolving after that, which broke both `--help` and re-invoking self. -SELF="${SCRIPT_DIR}/$(basename "${BASH_SOURCE[0]}")" - -INVOCATION_DIR="${PWD}" abspath() { case "$1" in /*) printf '%s\n' "$1" ;; *) printf '%s\n' "${INVOCATION_DIR%/}/$1" ;; esac; } - cd "${SCRIPT_DIR}" -FLAME=0 +FEATURES="" +FLAGS=() SCN_ARGS=() WALLETS_ARG="" while [ $# -gt 0 ]; do case "$1" in - --flame) FLAME=1; shift ;; - --wallets) WALLETS_ARG="${2:?--wallets needs a file path}"; shift 2 ;; - -h | --help) sed -n '3,24p' "${SELF}"; exit 0 ;; + --flame) FEATURES="${FEATURES} cpu-profile"; FLAGS+=("$1"); shift ;; + --memory-snapshot) FEATURES="${FEATURES} heap-profile"; FLAGS+=("$1"); shift ;; + --wallets) WALLETS_ARG="$(abspath "${2:?--wallets needs a file path}")"; FLAGS+=("$1" "${WALLETS_ARG}"); shift 2 ;; + -h | --help) sed -n '3,/^set /p' "${SELF}" | sed '$d'; exit 0 ;; -*) echo "unknown flag: $1" >&2; exit 1 ;; *) SCN_ARGS+=("$1"); shift ;; esac done -[ "${#SCN_ARGS[@]}" -gt 0 ] || { echo "usage: $0 ... [--flame] [--wallets ]" >&2; exit 1; } +[ "${#SCN_ARGS[@]}" -gt 0 ] || { echo "usage: $0 ... [--flame] [--memory-snapshot] [--wallets ]" >&2; exit 1; } -# Resolve each argument to scenario files. -# -# A bare path stays a bare path, so the original single-file invocation is -# untouched. Anything that is not a file is treated as a glob — which covers -# both a quoted pattern (`'scenarios/mainnet.*'`, expanded here) and one the -# caller's shell already expanded into several arguments. Patterns are tried -# against the invocation directory first and then against `scenarios/`, so -# `mainnet.*` works from anywhere. SCN_FILES=() +add_matches() { + local m + while IFS= read -r m; do + case "${m}" in /*) ;; *) m="$1/${m}" ;; esac + if [ -f "${m}" ]; then SCN_FILES+=("${m}"); fi + done < <(cd "$1" && compgen -G "$2") +} for arg in "${SCN_ARGS[@]}"; do - if [ -f "${arg}" ] || [ -f "$(abspath "${arg}")" ]; then - SCN_FILES+=("$(abspath "${arg}")") - continue - fi - matched=0 - while IFS= read -r hit; do - [ -f "${hit}" ] || continue - SCN_FILES+=("${hit}") - matched=1 - done < <(cd "${INVOCATION_DIR}" 2>/dev/null && shopt -s nullglob && printf '%s\n' ${arg} | while IFS= read -r m; do abspath "${m}"; done - cd "${SCRIPT_DIR}/scenarios" 2>/dev/null && shopt -s nullglob && printf '%s\n' ${arg} | while IFS= read -r m; do printf '%s\n' "${SCRIPT_DIR}/scenarios/${m}"; done) - [ "${matched}" -eq 1 ] || { echo "Error: no scenario matched: ${arg}" >&2; exit 1; } + n="${#SCN_FILES[@]}" + add_matches "${INVOCATION_DIR}" "${arg}" + [ "${#SCN_FILES[@]}" -gt "${n}" ] || add_matches "${SCRIPT_DIR}/scenarios" "${arg}" + [ "${#SCN_FILES[@]}" -gt "${n}" ] || { echo "Error: no scenario matched: ${arg}" >&2; exit 1; } done -# Run each scenario through a fresh invocation of this script and collect the -# numbers. Re-invoking rather than looping in place keeps the per-scenario path -# — exports, compose, traps, teardown — byte for byte what a single-file run has -# always done, so the extension cannot change the thing it is measuring. -# -# Taken for one scenario as much as for twenty: naming a file and naming a glob -# that happens to match one file are the same request, and having them leave -# their results in different shapes is a trap for anyone scripting on top. The -# child is marked so it runs the scenario instead of wrapping it again. +# Each scenario runs in a fresh invocation of this script, so one that fails +# cannot take the others with it. if [ -z "${BENCH_BATCH:-}" ]; then RUN_TS="$(date +%Y%m%d-%H%M%S)" OUT_DIR="${SCRIPT_DIR}/results/${RUN_TS}" @@ -117,11 +70,6 @@ if [ -z "${BENCH_BATCH:-}" ]; then TSV="${OUT_DIR}/results.tsv" REPORT="${OUT_DIR}/report.md" printf 'scenario\tcompleted\ttotal_ms\tblock_headers_ms\tfilter_headers_ms\tfilters_ms\ttransactions\tconfirmed_sat\tpeak_rss_mib\n' >"${TSV}" - - child_flags=() - [ "${FLAME}" -eq 1 ] && child_flags+=(--flame) - [ -n "${WALLETS_ARG}" ] && child_flags+=(--wallets "${WALLETS_ARG}") - metric() { awk -F':[[:space:]]*' -v k="$2" '$1==k {gsub(/[[:space:]]+$/,"",$2); print $2; exit}' "$1"; } echo "==> ${#SCN_FILES[@]} scenario(s); results in ${OUT_DIR}" @@ -129,37 +77,25 @@ if [ -z "${BENCH_BATCH:-}" ]; then name="$(basename "${f}" .yml)" log="${OUT_DIR}/${name}.log" echo "===== ${name} =====" - # A scenario that fails must not take the batch with it: record it and move - # on, or one bad run costs every result after it. BENCH_BATCH=1 BENCH_ARCHIVE_DIR="${OUT_DIR}" BENCH_ARCHIVE_NAME="${name}" \ - "${SELF}" "${f}" "${child_flags[@]+"${child_flags[@]}"}" 2>&1 | tee "${log}" || true - # Read the metrics from the summary the child archived, not from its stdout: - # the summary is written by the binary and is plain text either way, while a - # pty-captured log carries the bars' escape codes and carriage returns. - src="${log}" - if [ -s "${OUT_DIR}/${name}.summary.txt" ]; then src="${OUT_DIR}/${name}.summary.txt"; fi - wallet_line="$(grep -m1 -o 'confirmed_sat=[0-9]*' "${src}" || true)" - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ - "${name}" \ - "$(metric "${src}" completed)" \ - "$(metric "${src}" total_ms)" \ - "$(metric "${src}" block_headers_ms)" \ - "$(metric "${src}" filter_headers_ms)" \ - "$(metric "${src}" filters_ms)" \ - "$(metric "${src}" transactions)" \ - "$(sed -n 's/.*confirmed_sat=\([0-9]*\).*/\1/p' <<<"${wallet_line}")" \ - "$(metric "${src}" peak_rss_mib)" \ - >>"${TSV}" + "${SELF}" "${f}" ${FLAGS[@]+"${FLAGS[@]}"} 2>&1 | tee "${log}" || true + src="${OUT_DIR}/${name}.summary.txt" + [ -s "${src}" ] || src="${log}" + row="${name}" + for key in completed total_ms block_headers_ms filter_headers_ms filters_ms transactions; do + row+=$'\t'"$(metric "${src}" "${key}")" + done + row+=$'\t'"$(grep -m1 -o 'confirmed_sat=[0-9]*' "${src}" | cut -d= -f2 || true)" + row+=$'\t'"$(metric "${src}" peak_rss_mib)" + echo "${row}" >>"${TSV}" done { echo "# dash-spv bench — ${RUN_TS}" echo - echo "| scenario | completed | total_ms | headers_ms | filter_headers_ms | filters_ms | transactions | confirmed_sat | peak_rss_mib |" - echo "|---|---|---|---|---|---|---|---|---|" - tail -n +2 "${TSV}" | awk -F'\t' '{printf "| %s | %s | %s | %s | %s | %s | %s | %s | %s |\n", $1,$2,$3,$4,$5,$6,$7,$8,$9}' + awk -F'\t' '{ line = "|"; for (i = 1; i <= NF; i++) line = line " " $i " |"; print line + if (NR == 1) { line = "|"; for (i = 1; i <= NF; i++) line = line "---|"; print line } }' "${TSV}" } >"${REPORT}" - echo echo "==> report: ${REPORT}" cat "${REPORT}" @@ -167,33 +103,27 @@ if [ -z "${BENCH_BATCH:-}" ]; then fi SCN_FILE="${SCN_FILES[0]}" -[ -f "${SCN_FILE}" ] || { echo "Error: scenario file not found: ${SCN_FILE}" >&2; exit 1; } -YQ_VERSION="v4.44.6" -ensure_yq() { - if command -v yq >/dev/null 2>&1; then YQ=yq; return 0; fi - local bin="${SCRIPT_DIR}/.bin/yq" - if [ ! -x "${bin}" ]; then +if command -v yq >/dev/null 2>&1; then + YQ=yq +else + YQ="${SCRIPT_DIR}/.bin/yq" + if [ ! -x "${YQ}" ]; then + os="$(uname -s | tr '[:upper:]' '[:lower:]')" + case "$(uname -m)" in x86_64 | amd64) arch=amd64 ;; aarch64 | arm64) arch=arm64 ;; *) arch="$(uname -m)" ;; esac + echo "==> fetching yq into .bin/yq" mkdir -p "${SCRIPT_DIR}/.bin" - local os arch - os="$(uname -s | tr '[:upper:]' '[:lower:]')"; arch="$(uname -m)" - case "${arch}" in x86_64 | amd64) arch=amd64 ;; aarch64 | arm64) arch=arm64 ;; esac - echo "==> fetching yq ${YQ_VERSION} (${os}/${arch}) into .bin/yq" - curl -fsSL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_${os}_${arch}" \ - -o "${bin}" || { echo "Error: could not download yq; install it manually." >&2; exit 1; } - chmod +x "${bin}" + curl -fsSL "https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_${os}_${arch}" -o "${YQ}" \ + || { echo "Error: could not download yq; install it manually." >&2; exit 1; } + chmod +x "${YQ}" fi - YQ="${bin}" -} -ensure_yq -scn() { "${YQ}" "$1" "${SCN_FILE}"; } # evaluate a yq expression against the scenario file - -_peer_group() { - "${YQ}" ".peers[$1] | [.count, .latency_ms // 0, .jitter_ms // 0, .loss_pct // 0, .rate_kbit // 0, .corrupt_pct // 0, .reorder_pct // 0] | @tsv" "${SCN_FILE}" -} +fi +scn() { "${YQ}" "$1" "${SCN_FILE}"; } -build_netem() { - local lat="$1" jit="$2" loss="$3" rate="$4" corrupt="$5" reorder="$6" a="" +# `tc netem` arguments for the link described at yq path $1. +netem_args() { + local lat jit loss rate corrupt reorder a="" + read -r lat jit loss rate corrupt reorder <<<"$(scn "$1 | [.latency_ms // 0, .jitter_ms // 0, .loss_pct // 0, .rate_kbit // 0, .corrupt_pct // 0, .reorder_pct // 0] | @tsv")" [ "${lat}" != 0 ] && { a="delay ${lat}ms"; [ "${jit}" != 0 ] && a="${a} ${jit}ms"; } [ "${loss}" != 0 ] && a="${a} loss ${loss}%" [ "${rate}" != 0 ] && a="${a} rate ${rate}kbit" @@ -202,38 +132,8 @@ build_netem() { echo "${a# }" } -# The measured client's own link shaping, if the scenario asks for one -client_netem() { - local lat jit loss rate corrupt reorder - read -r lat jit loss rate corrupt reorder <<<"$("${YQ}" \ - '[.client.latency_ms // 0, .client.jitter_ms // 0, .client.loss_pct // 0, .client.rate_kbit // 0, .client.corrupt_pct // 0, .client.reorder_pct // 0] | @tsv' \ - "${SCN_FILE}")" - build_netem "${lat}" "${jit}" "${loss}" "${rate}" "${corrupt}" "${reorder}" -} - -# The client's bandwidth cap on its own, so the container can mirror it onto -# ingress -client_rate_kbit() { - "${YQ}" '.client.rate_kbit // 0' "${SCN_FILE}" -} - -peers_summary() { - local ng g count lat jit loss rate corrupt reorder tag out="" - ng="$(scn '.peers | length')" - case "${ng}" in ''|null|*[!0-9]*) ng=0 ;; esac - for ((g = 0; g < ng; g++)); do - read -r count lat jit loss rate corrupt reorder <<<"$(_peer_group "${g}")" - tag="${lat}ms" - [ "${jit}" != 0 ] && tag="${tag}±${jit}" - [ "${loss}" != 0 ] && tag="${tag}/${loss}%loss" - [ "${rate}" != 0 ] && tag="${tag}/${rate}kbit" - out="${out}, ${count}×${tag}" - done - echo "${out#, }" -} - emit_compose() { - local out="$1" peer_cpus="$2" + local out="$1" groups g count n peer=0 cat >"${out}" <
>"${out}" <>"${out}" </dev/null \ + if [ "\$\${INGRESS_RATE_KBIT:-0}" != 0 ]; then + ip link add ifb0 type ifb \ && ip link set ifb0 up \ && tc qdisc add dev eth0 handle ffff: ingress \ && tc filter add dev eth0 parent ffff: protocol all prio 1 u32 \ match u32 0 0 action mirred egress redirect dev ifb0 \ - && tc qdisc add dev ifb0 root netem rate \$\${INGRESS_RATE_KBIT}kbit; then - echo "client netem (ingress): rate \$\${INGRESS_RATE_KBIT}kbit via ifb0" - else - tc qdisc del dev eth0 ingress 2>/dev/null || true - if tc qdisc add dev eth0 handle ffff: ingress 2>/dev/null \ - && tc filter add dev eth0 parent ffff: protocol all prio 1 u32 \ - match u32 0 0 action police rate \$\${INGRESS_RATE_KBIT}kbit \ - burst \$\${INGRESS_BURST_K}k mtu 64k conform-exceed drop; then - echo "client police (ingress): rate \$\${INGRESS_RATE_KBIT}kbit burst \$\${INGRESS_BURST_K}k" - echo "NOTE: no ifb, so ingress is POLICED (drops) not shaped (queues) — throughput lands within ~3% but the loss pattern differs; do not compare against ifb runs" - else - echo "WARNING: ingress shaping failed (NET_ADMIN? no ifb and no act_police?)" - echo "WARNING: DOWNLOADS ARE UNSHAPED — rate_kbit is NOT enforced and this run is not comparable" - fi - fi + && tc qdisc add dev ifb0 root netem rate \$\${INGRESS_RATE_KBIT}kbit \ + || { echo "ERROR: cannot shape the client's downloads (no ifb in this kernel?)"; exit 1; } + echo "client netem (ingress): rate \$\${INGRESS_RATE_KBIT}kbit via ifb0" fi exec /usr/local/bin/dash-spv-bench SERVICE } - MODE="$(scn '.mode // "local"')" case "${MODE}" in local | testnet | mainnet) ;; *) echo "Error: mode must be 'local', 'testnet' or 'mainnet' (got '${MODE}')" >&2; exit 1 ;; esac -export BENCH_MODE="${MODE}" export CLONE_DIR="${CLONE_DIR:-/nonexistent}" -CLIENT_NETEM="$(client_netem)" -CLIENT_RATE_KBIT="$(client_rate_kbit)" +CLIENT_NETEM="$(netem_args .client)" +CLIENT_RATE_KBIT="$(scn '.client.rate_kbit // 0')" case "${CLIENT_RATE_KBIT}" in '' | *[!0-9]*) echo "Error: client.rate_kbit must be a whole number of kbit (got '${CLIENT_RATE_KBIT}')" >&2; exit 1 ;; esac -# Burst for the policer fallback: ~0.5s of the rate (kbit/16 => kbytes), floored -# so a very small rate still gets a workable bucket. Unused on the ifb path. -CLIENT_INGRESS_BURST_K=$(( CLIENT_RATE_KBIT / 16 )) -if [ "${CLIENT_INGRESS_BURST_K}" -lt 32 ]; then CLIENT_INGRESS_BURST_K=32; fi BENCH_CPUS="$(scn '.cpus // ""')" -BENCH_MAX_PEERS="$(scn '.max_peers // ""')" # only if set; else the ClientConfig default +BENCH_MAX_PEERS="$(scn '.max_peers // ""')" export BENCH_MAX_PEERS - -BENCH_WALLET_FILE="${SCRIPT_DIR}/wallets.txt" -[ -n "${WALLETS_ARG}" ] && BENCH_WALLET_FILE="$(abspath "${WALLETS_ARG}")" -export BENCH_WALLET_FILE +BENCH_WALLET_FILE="${WALLETS_ARG:-${SCRIPT_DIR}/wallets.txt}" export BENCH_STORAGE_DIR="${SCRIPT_DIR}/bench-storage" DESC="$(scn '.description // ""')" -[ -n "${DESC}" ] && echo "==> description: ${DESC}" +[ -z "${DESC}" ] || echo "==> description: ${DESC}" -BLOCKS="" if [ "${MODE}" = local ]; then - BLOCKS="$(scn '.blocks // 1000000')" - export BENCH_HEIGHT="${BLOCKS}" - unset BENCH_START_HEIGHT # local always syncs from genesis + BENCH_HEIGHT="$(scn '.blocks // 1000000')" + export BENCH_HEIGHT + unset BENCH_START_HEIGHT else BENCH_PEERS="$(scn '.peers // [] | join(",")')" export BENCH_PEERS - sh="$(scn '.start_height // ""')"; [ -n "${sh}" ] && export BENCH_START_HEIGHT="${sh}" + start_height="$(scn '.start_height // ""')" + [ -z "${start_height}" ] || export BENCH_START_HEIGHT="${start_height}" fi -expand_cpus() { - local part lo hi - local IFS=, - for part in $1; do - case "${part}" in - *-*) lo="${part%-*}"; hi="${part#*-}"; seq "${lo}" "${hi}" ;; - *) echo "${part}" ;; - esac - done -} - -CPU_PREFIX=() +# The client gets BENCH_CPUS, docker peers get every other core. +BENCH_PEER_CPUS="" if [ -n "${BENCH_CPUS}" ]; then ncpu="$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 0)" - bench_cores="$(expand_cpus "${BENCH_CPUS}" | sort -nu)" - if [ "${ncpu}" -gt 0 ]; then - peer_cores="" - for i in $(seq 0 $((ncpu - 1))); do - grep -qxF "${i}" <<<"${bench_cores}" || peer_cores="${peer_cores},${i}" - done - BENCH_PEER_CPUS="${peer_cores#,}" - fi - - if [ -n "${CLIENT_NETEM}" ]; then - # The client is a container: `cpuset` on the service does the pinning that - # `taskset` does for a host run, and it works on macOS where taskset does - # not exist at all. Emitted by `emit_client_service` from BENCH_CPUS. - echo "==> pinning the measured client container to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" - elif command -v taskset >/dev/null 2>&1; then - CPU_PREFIX=(taskset -c "${BENCH_CPUS}") - echo "==> pinning the measured run to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" - else - echo "==> note: 'taskset' not found (e.g. macOS); running the measured binary UNPINNED${BENCH_PEER_CPUS:+ (docker peers still pinned to ${BENCH_PEER_CPUS})}" >&2 - fi -fi - -# Where the measured binary will be, decided before the compose that mounts -# it is written. Everything lands in the workspace `target/`: a cross build -# gets cargo's own per-triple subdirectory, so it cannot collide with a host -# build, and neither duplicates the dependency graph. -if [ -n "${CLIENT_NETEM}" ] && [ "$(uname -s)" != Linux ]; then - CROSS_TRIPLE="$(docker run --rm "${RUST_IMAGE}" rustc -vV | sed -n 's/^host: //p')" - [ -n "${CROSS_TRIPLE}" ] || { echo "could not read the build image's target triple" >&2; exit 1; } - BIN="${REPO_ROOT}/target/${CROSS_TRIPLE}/release/dash-spv-bench" -else - CROSS_TRIPLE="" - BIN="${REPO_ROOT}/target/release/dash-spv-bench" -fi - -# A compose file is needed for the peers (local mode) and for the client -# container (any mode with a `client:` section) — testnet/mainnet shape the -# client against the real network, with no peer services at all. -if [ "${MODE}" = local ] || [ -n "${CLIENT_NETEM}" ]; then - COMPOSE_FILE="$(mktemp "${SCRIPT_DIR}/.scenario.XXXXXX")" - mv "${COMPOSE_FILE}" "${COMPOSE_FILE}.yml" - COMPOSE_FILE="${COMPOSE_FILE}.yml" - npeers="$(emit_compose "${COMPOSE_FILE}" "${BENCH_PEER_CPUS:-}")" - if [ "${MODE}" = local ]; then - echo "==> scenario '$(basename "${SCN_FILE}" .yml)': ${npeers} peers [$(peers_summary)], cpus=${BENCH_CPUS:-}, blocks=${BLOCKS}" - fi + bench_cores="$(IFS=,; for part in ${BENCH_CPUS}; do case "${part}" in *-*) seq "${part%-*}" "${part#*-}" ;; *) echo "${part}" ;; esac; done)" + for ((i = 0; i < ncpu; i++)); do + grep -qxF "${i}" <<<"${bench_cores}" || BENCH_PEER_CPUS="${BENCH_PEER_CPUS},${i}" + done + BENCH_PEER_CPUS="${BENCH_PEER_CPUS#,}" + echo "==> pinning the client container to CPUs ${BENCH_CPUS}${BENCH_PEER_CPUS:+; docker peers to ${BENCH_PEER_CPUS}}" fi -[ -n "${CLIENT_NETEM}" ] && echo "==> client link shaped: ${CLIENT_NETEM}" +# Built in the Rust image so the binary links against the client image's glibc. +BIN="${REPO_ROOT}/target/bench/release/dash-spv-bench" +echo "==> building bench binary in ${RUST_IMAGE}" +docker run --rm -v "${REPO_ROOT}:/src" -w /src --user "$(id -u):$(id -g)" \ + -e CARGO_HOME=/src/target/bench/cargo-home -e CARGO_TARGET_DIR=/src/target/bench \ + -e CARGO_NET_GIT_FETCH_WITH_CLI=true -e CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ + "${RUST_IMAGE}" cargo build --release -p dash-spv-bench ${FEATURES:+--features "${FEATURES# }"} +[ -x "${BIN}" ] || { echo "build produced no binary at ${BIN}" >&2; exit 1; } + +COMPOSE_FILE="$(mktemp "${SCRIPT_DIR}/.scenario.XXXXXX")" +mv "${COMPOSE_FILE}" "${COMPOSE_FILE}.yml" +COMPOSE_FILE="${COMPOSE_FILE}.yml" +emit_compose "${COMPOSE_FILE}" +[ -z "${CLIENT_NETEM}" ] || echo "==> client link shaped: ${CLIENT_NETEM}" compose() { docker compose -p "${PROJECT}" -f "${COMPOSE_FILE}" "$@"; } +teardown() { + compose down --remove-orphans >/dev/null 2>&1 || true + [ ! -f "${STATE}" ] || rm -rf "$(cat "${STATE}")" "${STATE}" 2>/dev/null || true + rm -rf "${SCRIPT_DIR}"/.bench-clones.* 2>/dev/null || true +} +trap 'teardown; rm -f "${COMPOSE_FILE}"' EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +trap 'exit 129' HUP + wait_loaded() { - local c logs + local c logs all for _ in $(seq 1 400); do - local all=1 + all=1 for c in "$@"; do logs="$(docker logs "${c}" 2>&1)" || { all=0; break; } case "${logs}" in *"init message: Done loading"*) ;; *) all=0; break ;; esac done - [ "${all}" -eq 1 ] && return 0 + [ "${all}" -eq 0 ] || return 0 echo -n "."; sleep 3 done return 1 } -teardown() { - [ -n "${COMPOSE_FILE}" ] && compose down --remove-orphans >/dev/null 2>&1 || true - [ -f "${STATE}" ] && { rm -rf "$(cat "${STATE}")" 2>/dev/null || true; rm -f "${STATE}"; } - rm -rf "${SCRIPT_DIR}"/.bench-clones.* 2>/dev/null || true -} - -arm_teardown() { - # On exit: tear down, THEN delete the generated compose (down needs it to still exist). - trap 'teardown; [ -n "${COMPOSE_FILE}" ] && rm -f "${COMPOSE_FILE}"' EXIT - trap 'exit 130' INT - trap 'exit 143' TERM - trap 'exit 129' HUP -} - -bring_up() { +if [ "${MODE}" = local ]; then + bash "${SCRIPT_DIR}/snapshot-chain.sh" # builds ./chain-data to BENCH_HEIGHT via docker echo "==> ensuring a clean network" teardown docker image inspect "${IMAGE}" >/dev/null 2>&1 || { echo "==> building peer image"; compose build; } - - # The client is in the same compose file but is not a peer: it must not be - # counted, started here, or waited on for "Done loading". - local services; services="$(compose config --services | grep -v '^client$' || true)" - local n; n="$(echo ${services} | wc -w | tr -d ' ')" - # BENCH_PEERS is derived from the generated peers (host ports 19401..). - local peers_csv="" - for svc in ${services}; do peers_csv="${peers_csv},127.0.0.1:$((19400 + ${svc#dashd}))"; done - export BENCH_PEERS="${peers_csv#,}" - - echo "==> CoW-cloning ${CHAIN_DIR} for ${n} peers (instant)" - local clone_dir; clone_dir="$(mktemp -d "${SCRIPT_DIR}/.bench-clones.XXXXXX")" - echo "${clone_dir}" > "${STATE}" + services="$(compose config --services | grep -v '^client$' || true)" + npeers="$(echo ${services} | wc -w | tr -d ' ')" + peers_summary="$(scn '[.peers[] | (.count | tostring) + "×" + ([to_entries[] | select(.key != "count") | .key + "=" + (.value | tostring)] | join(" "))] | join(", ")')" + echo "==> scenario '$(basename "${SCN_FILE}" .yml)': ${npeers} peers [${peers_summary}], cpus=${BENCH_CPUS:-}, blocks=${BENCH_HEIGHT}" + echo "==> CoW-cloning ${CHAIN_DIR} for ${npeers} peers" + clone_dir="$(mktemp -d "${SCRIPT_DIR}/.bench-clones.XXXXXX")" + echo "${clone_dir}" >"${STATE}" for svc in ${services}; do - local dst="${clone_dir}/peer${svc#dashd}" + dst="${clone_dir}/peer${svc#dashd}" cp -c -R "${CHAIN_DIR}" "${dst}" 2>/dev/null \ || cp --reflink=auto -R "${CHAIN_DIR}" "${dst}" 2>/dev/null \ || cp -R "${CHAIN_DIR}" "${dst}" done - - local batch=4 - echo "==> starting ${n} peers in batches of ${batch}" - local started="" count=0 + echo "==> starting ${npeers} peers in batches of 4" + started="" + count=0 for svc in ${services}; do CLONE_DIR="${clone_dir}" compose up -d "${svc}" >/dev/null 2>&1 - started="${started} spv-bench-${svc}"; count=$((count + 1)) - if [ $((count % batch)) -eq 0 ]; then - echo -n " loaded ${count}/${n} " + started="${started} spv-bench-${svc}" + count=$((count + 1)) + if [ $((count % 4)) -eq 0 ] || [ "${count}" -eq "${npeers}" ]; then + echo -n " loaded ${count}/${npeers} " wait_loaded ${started} || { echo " timeout loading batch" >&2; exit 1; } echo " ok" fi done - if [ $((count % batch)) -ne 0 ]; then # wait on the trailing partial batch too - echo -n " loaded ${count}/${n} " - wait_loaded ${started} || { echo " timeout loading batch" >&2; exit 1; } - echo " ok" - fi - - echo "==> ${n} peers started" - - # A containerised client shares the compose network with the peers, so it must - # reach them at their container addresses; the published host ports only exist - # for a client running on the host. Read the addresses back from the running - # containers rather than pinning a subnet in the compose file: a pinned subnet - # is one more thing that can collide with whatever else the machine has up, - # VPNs included, and it buys nothing a lookup does not. - if [ -n "${CLIENT_NETEM}" ]; then - local ip csv="" - for svc in ${services}; do - ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "spv-bench-${svc}")" - [ -n "${ip}" ] || { echo "Error: could not resolve address of spv-bench-${svc}" >&2; exit 1; } - csv="${csv},${ip}:19400" - done - export BENCH_PEERS="${csv#,}" - echo "==> client will reach peers at ${BENCH_PEERS}" - fi -} - -build_bin() { - if [ -n "${CROSS_TRIPLE}" ]; then - # The client runs in a Linux container, so the binary has to be a Linux - # one. On a Linux host the ordinary build already is — same triple, and - # glibc is forward compatible — so only a non-Linux host comes here. - # Bind-mounts the workspace and builds into its `target/`, which cargo - # keeps under the triple, so this stays incremental — an image layer build - # would replay the whole workspace on every source change, which makes A/B - # runs unusable. - echo "==> cross-building bench binary (${RUST_IMAGE}, ${CROSS_TRIPLE})" - # As the invoking user, with CARGO_HOME inside the workspace: writing - # into the shared `target/` as root leaves artifacts the host build then - # cannot overwrite. - docker run --rm \ - -v "${REPO_ROOT}:/src" \ - -w /src \ - --user "$(id -u):$(id -g)" \ - -e CARGO_HOME=/src/target/.cross-cargo-home \ - -e CARGO_TARGET_DIR=/src/target \ - -e CARGO_NET_GIT_FETCH_WITH_CLI=true \ - -e CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ - "${RUST_IMAGE}" \ - cargo build --release --target "${CROSS_TRIPLE}" -p dash-spv-bench - else - echo "==> building bench binary (release + line-table symbols)" - ( cd "${REPO_ROOT}" && CARGO_PROFILE_RELEASE_DEBUG=line-tables-only \ - cargo build --release -p dash-spv-bench ) - fi - [ -x "${BIN}" ] || { echo "build produced no binary at ${BIN}" >&2; exit 1; } -} - -FLAME_TOOL="" -if [ "${FLAME}" -eq 1 ]; then # fail fast on missing profiler deps, before building - if command -v perf >/dev/null; then FLAME_TOOL=perf # Linux - elif command -v /usr/bin/sample >/dev/null; then FLAME_TOOL=sample # macOS - else echo "flame mode needs 'perf' (Linux) or '/usr/bin/sample' (macOS)" >&2; exit 1; fi - [ -f "${FLAMEGRAPH_DIR}/flamegraph.pl" ] || \ - git clone --depth 1 https://github.com/brendangregg/FlameGraph "${FLAMEGRAPH_DIR}" -fi - -build_bin - -if [ "${MODE}" = local ]; then - arm_teardown - bash "${SCRIPT_DIR}/snapshot-chain.sh" # builds ./chain-data to BENCH_HEIGHT via docker - bring_up + # Container addresses, read back rather than pinned to a subnet that could + # collide with whatever else the machine has up. + csv="" + for svc in ${services}; do + ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "spv-bench-${svc}")" + [ -n "${ip}" ] || { echo "Error: could not resolve address of spv-bench-${svc}" >&2; exit 1; } + csv="${csv},${ip}:19400" + done + export BENCH_PEERS="${csv#,}" + echo "==> ${npeers} peers started, reachable at ${BENCH_PEERS}" else echo "==> ${MODE} mode, peers: ${BENCH_PEERS:-}" fi -# Run the sync with its live output straight on the terminal instead of through -# the batch wrapper's pipe -run_live() { - # Not `[ -w /dev/tty ]`: the device node is writable even with no - # controlling terminal, and the redirect then fails with ENXIO. Only - # actually opening it answers the question. - if { : >/dev/tty; } 2>/dev/null; then - "$@" >/dev/tty 2>&1 - else - "$@" - fi -} - -if [ -n "${CLIENT_NETEM}" ]; then - # `run` rather than `up`: it streams the client's own stdout and gives back - # its exit code, which is what the report is read from. Profiling is not - # wired through the container, so `--flame` is refused rather than silently - # producing a graph of the host doing nothing. - [ "${FLAME}" -eq 0 ] || { echo "Error: --flame is not supported with a containerised client" >&2; exit 1; } - arm_teardown - echo "==> running sync in the client container" - run_live compose run --rm --build client -elif [ "${FLAME}" -eq 0 ]; then - echo "==> running sync" - run_live "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" -elif [ "${FLAME_TOOL}" = perf ]; then - echo "==> running sync under perf" - mkdir -p "$(dirname "${FLAME_SVG}")" - perf record -F 499 -g -o /tmp/bench-perf.data -- "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" - perf script -i /tmp/bench-perf.data \ - | "${FLAMEGRAPH_DIR}/stackcollapse-perf.pl" \ - | "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "dash-spv sync" --colors hot > "${FLAME_SVG}" - echo "==> wrote ${FLAME_SVG}" +echo "==> running sync in the client container" +if { : >/dev/tty; } 2>/dev/null; then + compose run --rm --build client >/dev/tty 2>&1 else - echo "==> running sync under the sampler" - "${CPU_PREFIX[@]+"${CPU_PREFIX[@]}"}" "${BIN}" & bpid=$! - sleep 3 - /usr/bin/sample "${bpid}" 2000 1 -file /tmp/bench.sample.txt -mayDie >/dev/null 2>&1 || true - wait "${bpid}" 2>/dev/null || true - mkdir -p "$(dirname "${FLAME_SVG}")" - "${FLAMEGRAPH_DIR}/stackcollapse-sample.awk" /tmp/bench.sample.txt \ - | sed -E 's/^Thread_[^;]*;//' \ - | "${FLAMEGRAPH_DIR}/flamegraph.pl" --title "dash-spv sync" --colors hot > "${FLAME_SVG}" - echo "==> wrote ${FLAME_SVG}" + compose run --rm --build client fi -# Keep this run's outputs, which `bench-storage/` does not: it is wiped at the -# start of every run, so without this the trace of anything but the most recent -# scenario is gone — and in a batch that meant 15 of 16 syncs left nothing to -# look at afterwards. -# -# The batch passes its own directory in, so a child archives straight into it -# and there is one copy, named after the scenario. A lone run makes its own. -ARCHIVE_DIR="${BENCH_ARCHIVE_DIR:-${SCRIPT_DIR}/results/$(date +%Y%m%d-%H%M%S)-$(basename "${SCN_FILE}" .yml)}" -# Prefixed with the scenario only inside a batch, where one directory holds -# every scenario. A lone run's directory is already named after it. -ARCHIVE_NAME="${BENCH_ARCHIVE_NAME:-}" +ARCHIVE_DIR="${BENCH_ARCHIVE_DIR:?BENCH_ARCHIVE_DIR is set by the batch wrapper}" mkdir -p "${ARCHIVE_DIR}" -for out in run.log summary.txt; do +for out in run.log summary.txt flamegraph.svg heap-peak.svg; do [ -s "${BENCH_STORAGE_DIR}/${out}" ] || continue - cp "${BENCH_STORAGE_DIR}/${out}" "${ARCHIVE_DIR}/${ARCHIVE_NAME:+${ARCHIVE_NAME}.}${out}" + cp "${BENCH_STORAGE_DIR}/${out}" "${ARCHIVE_DIR}/${BENCH_ARCHIVE_NAME}.${out}" done echo "==> logs archived to ${ARCHIVE_DIR}" diff --git a/dash-spv-bench/src/main.rs b/dash-spv-bench/src/main.rs index 03b34da7c..35544319b 100644 --- a/dash-spv-bench/src/main.rs +++ b/dash-spv-bench/src/main.rs @@ -1,5 +1,7 @@ mod dashboard; mod metrics; +#[cfg(target_os = "linux")] +mod profile; use std::net::SocketAddr; use std::path::PathBuf; @@ -40,11 +42,11 @@ fn load_mnemonics() -> Vec { .unwrap_or_default() } -fn peak_rss_kb() -> Option { +fn proc_status_kb(field: &str) -> Option { std::fs::read_to_string("/proc/self/status") .ok()? .lines() - .find_map(|line| line.strip_prefix("VmHWM:"))? + .find_map(|line| line.strip_prefix(field))? .split_whitespace() .next()? .parse() @@ -89,6 +91,9 @@ async fn main() -> Result<()> { ) .init(); + #[cfg(target_os = "linux")] + let profilers = profile::start(); + let mode = env_or("BENCH_MODE", "local").trim().to_ascii_lowercase(); let (network, remote) = match mode.as_str() { "local" => (Network::Testnet, false), @@ -197,9 +202,13 @@ async fn main() -> Result<()> { run_handle.abort(); let _ = run_handle.await; + let peak_rss_kb = proc_status_kb("VmHWM:"); + #[cfg(target_os = "linux")] + profilers.finish(&output_dir); + use std::fmt::Write as _; let mut report = format!("{m}\n"); - if let Some(kb) = peak_rss_kb() { + if let Some(kb) = peak_rss_kb { let _ = writeln!(report, "peak_rss_mib: {}", kb / 1024); } diff --git a/dash-spv-bench/src/profile.rs b/dash-spv-bench/src/profile.rs new file mode 100644 index 000000000..05735115a --- /dev/null +++ b/dash-spv-bench/src/profile.rs @@ -0,0 +1,91 @@ +use std::path::Path; + +#[cfg(feature = "heap-profile")] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[cfg(feature = "heap-profile")] +#[export_name = "_rjem_malloc_conf"] +pub static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; + +#[cfg(feature = "heap-profile")] +type HeapPeak = std::sync::Arc>>; + +pub(crate) struct Profilers { + #[cfg(feature = "cpu-profile")] + cpu: Option>, + #[cfg(feature = "heap-profile")] + heap_peak: HeapPeak, +} + +pub(crate) fn start() -> Profilers { + let profilers = Profilers { + #[cfg(feature = "cpu-profile")] + cpu: pprof::ProfilerGuardBuilder::default() + .frequency(99) + .blocklist(&["libc", "libgcc", "pthread", "vdso"]) + .build() + .inspect_err(|e| tracing::warn!("cpu profiler not started: {e}")) + .ok(), + #[cfg(feature = "heap-profile")] + heap_peak: HeapPeak::default(), + }; + #[cfg(feature = "heap-profile")] + { + let peak = profilers.heap_peak.clone(); + std::thread::spawn(move || { + let Some(ctl) = jemalloc_pprof::PROF_CTL.as_ref() else { + tracing::warn!("jemalloc heap profiling is not enabled"); + return; + }; + let mut dumped_kb = 0; + while let Some(rss_kb) = crate::proc_status_kb("VmRSS:") { + if rss_kb > dumped_kb + dumped_kb / 20 { + match ctl.blocking_lock().dump_profile() { + Ok(profile) => { + dumped_kb = rss_kb; + if let Ok(mut slot) = peak.lock() { + *slot = Some((rss_kb, profile)); + } + } + Err(e) => { + tracing::warn!("heap snapshot failed: {e}"); + return; + } + } + } + std::thread::sleep(std::time::Duration::from_millis(500)); + } + }); + } + profilers +} + +impl Profilers { + #[allow(unused_variables)] + pub(crate) fn finish(self, dir: &Path) { + #[cfg(feature = "heap-profile")] + if let Some((rss_kb, profile)) = self.heap_peak.lock().ok().and_then(|mut slot| slot.take()) + { + let mut opts = jemalloc_pprof::FlamegraphOptions::default(); + opts.title = format!("dash-spv live heap at the RSS peak ({} MiB)", rss_kb / 1024); + opts.count_name = "bytes".to_string(); + let written = + profile.to_flamegraph(&mut opts).map_err(|e| e.to_string()).and_then(|svg| { + std::fs::write(dir.join("heap-peak.svg"), svg).map_err(|e| e.to_string()) + }); + if let Err(e) = written { + tracing::warn!("could not write heap-peak.svg: {e}"); + } + } + #[cfg(feature = "cpu-profile")] + if let Some(report) = self.cpu.as_ref().and_then(|guard| guard.report().build().ok()) { + let written = std::fs::File::create(dir.join("flamegraph.svg")) + .map_err(|e| e.to_string()) + .and_then(|file| report.flamegraph(file).map_err(|e| e.to_string())); + if let Err(e) = written { + tracing::warn!("could not write flamegraph.svg: {e}"); + } + } + } +}