From a1bd714f4ceacec2c231dc1318ed890e7a212d7f Mon Sep 17 00:00:00 2001 From: Jonathan Kashi Date: Wed, 2 Sep 2026 00:32:18 -0400 Subject: [PATCH] Selectivity-adaptive filtered search: sparse filters get exact results A sparse allow-bitmap makes graph traversal the wrong algorithm: too few eligible nodes enter the beam, pruning never engages, and the walk decays toward an expensive, approximate scan of the reachable graph. The provably right strategy at high selectivity is a direct scan of the allowed ids - cheaper AND exact. All three search modes now route their candidate pool through collect_pool(): when popcount(allow) <= 16*ef (the popcount is a microsecond pass, bits beyond capacity masked), a zero-allocation exact scan of the allowed set fills the pool - recall 1.0 over the filter by construction, with ascending-id iteration reproducing the brute-force (distance, id) tie order exactly. Larger filters keep the existing traversal with filtered-out nodes routing. Tests: sparse-filter results gated EQUAL to brute force (ids and distances), re-ranked modes stay within the filter, empty bitmap returns 0. Benchmark gains a filtered-search table across 0.1%/1%/10% selectivities measured against exact filtered ground truth. Windows: 6/6 suites green both configs; cross-platform validation via CI (local WSL is congested by unrelated workload load-avg ~30 today - measured numbers for the README follow from a detached run). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017ULiWeodALX2ZQiLKvTw25 --- include/edgevector/hnsw_graph.hpp | 88 ++++++++++++++++++++++++++++--- tests/benchmark_100k.cpp | 61 +++++++++++++++++++++ tests/test_hnsw_graph.cpp | 50 ++++++++++++++++++ 3 files changed, 193 insertions(+), 6 deletions(-) diff --git a/include/edgevector/hnsw_graph.hpp b/include/edgevector/hnsw_graph.hpp index e334125..bd85006 100644 --- a/include/edgevector/hnsw_graph.hpp +++ b/include/edgevector/hnsw_graph.hpp @@ -40,9 +40,17 @@ // memory. Requires the float query alongside the // quantized one. // Both accept an optional caller-owned allow-bitmap (bit id set = id may be -// returned) and always exclude soft-deleted nodes; filtered-out nodes still -// route traversal. NOTE: a highly selective filter degrades toward a scan of -// the reachable graph, as in every filtered-HNSW implementation. +// returned) and always exclude soft-deleted nodes. +// +// FILTERED SEARCH IS SELECTIVITY-ADAPTIVE. A sparse filter makes graph +// traversal the wrong algorithm: too few eligible nodes ever enter the beam, +// pruning never engages, and the walk decays toward scanning the reachable +// graph - expensively and approximately. So when the allowed population is +// small relative to the beam (popcount(allow) <= 16 * ef; the popcount is a +// ~microsecond pass over the bitmap), the search switches to a direct scan +// of the allowed ids, which is both cheaper AND exact - recall 1.0 over the +// allowed set by construction. Larger filters keep the graph traversal, with +// filtered-out nodes still routing. Both paths are zero-allocation. // // DELETION AND SLOT RECLAMATION // ----------------------------- @@ -631,7 +639,7 @@ class HNSWGraph { } clamp_kef(k, ef); - std::uint32_t found = beam_layer0(ctx, q, ef, allow); + std::uint32_t found = collect_pool(ctx, q, ef, allow); while (found > k) { // keep only the k best ctx.res_heap_.pop(); --found; @@ -671,7 +679,7 @@ class HNSWGraph { } clamp_kef(k, ef); - const std::uint32_t found = beam_layer0(ctx, q_bits, ef, allow); + const std::uint32_t found = collect_pool(ctx, q_bits, ef, allow); if (found == 0u) { return 0u; } @@ -738,7 +746,7 @@ class HNSWGraph { } clamp_kef(k, ef); - const std::uint32_t found = beam_layer0(ctx, q_bits, ef, allow); + const std::uint32_t found = collect_pool(ctx, q_bits, ef, allow); if (found == 0u) { return 0u; } @@ -1305,6 +1313,74 @@ class HNSWGraph { return search_layer(ctx, q, ep, ef, 0u, allow, true); } + // Population of an allow-bitmap over the valid id range (bits beyond + // capacity are masked off, whatever the caller left there). + std::uint64_t popcount_allow(const std::uint64_t* allow) const noexcept { + const std::size_t full = static_cast(capacity_) / 64u; + std::uint64_t total = 0u; + for (std::size_t w = 0u; w < full; ++w) { + total += static_cast(__builtin_popcountll(allow[w])); + } + const std::uint32_t rem = capacity_ & 63u; + if (rem != 0u) { + const std::uint64_t mask = (1ull << rem) - 1ull; + total += static_cast( + __builtin_popcountll(allow[full] & mask)); + } + return total; + } + + // Exact scan over the allowed ids: the optimal strategy for sparse + // filters. Fills ctx.res_heap_ with the ef best eligible candidates - + // EXACT over the allowed set, since every allowed id is scored. Ids are + // visited ascending and ties skip the later id, which reproduces the + // (distance, id) selection order of the brute-force baseline exactly. + // Zero allocation. + std::uint32_t filtered_scan(SearchContext& ctx, const std::uint8_t* q, + std::uint32_t ef, + const std::uint64_t* allow) const noexcept { + ctx.res_heap_.clear(); + const std::size_t words = + (static_cast(capacity_) + 63u) / 64u; + const std::uint32_t rem = capacity_ & 63u; + for (std::size_t w = 0u; w < words; ++w) { + std::uint64_t bits = allow[w]; + if (rem != 0u && w == words - 1u) { + bits &= (1ull << rem) - 1ull; + } + while (bits != 0ull) { + const std::uint64_t low = bits & (0ull - bits); + bits ^= low; + const std::uint32_t id = static_cast( + w * 64u + static_cast(__builtin_ctzll(low))); + if (levels_[id] == kNotInserted || is_deleted(id)) { + continue; + } + const std::uint32_t d = distance_to(q, id); + if (ctx.res_heap_.size() < ef) { + ctx.res_heap_.push(detail::Candidate{d, id}); + } else if (d < ctx.res_heap_.top().dist) { + ctx.res_heap_.push(detail::Candidate{d, id}); + ctx.res_heap_.pop(); + } + } + } + return ctx.res_heap_.size(); + } + + // The candidate-pool collector behind all three search modes: picks the + // scan for sparse filters (exact and cheaper), the beam otherwise. + std::uint32_t collect_pool(SearchContext& ctx, const std::uint8_t* q, + std::uint32_t ef, + const std::uint64_t* allow) const noexcept { + if (allow != nullptr && + popcount_allow(allow) <= + static_cast(ef) * 16u) { + return filtered_scan(ctx, q, ef, allow); + } + return beam_layer0(ctx, q, ef, allow); + } + // Beam search on one layer (Algorithm 2), with result-eligibility // filtering: ineligible nodes (tombstoned, or cleared in `allow`) still // route within the beam bound but never enter the result heap. Results diff --git a/tests/benchmark_100k.cpp b/tests/benchmark_100k.cpp index f0eb5cc..0fa1971 100644 --- a/tests/benchmark_100k.cpp +++ b/tests/benchmark_100k.cpp @@ -371,6 +371,67 @@ void run_scenario(const char* name, bool clustered, bool full_report) { } std::printf("\n"); std::fflush(stdout); + + // Filtered search across selectivities (allow every stride-th id). + // Recall is measured against the exact brute-force top-10 over the + // allowed set; sparse filters take the adaptive exact-scan path. + if (full_report) { + std::printf("Filtered search (ef = 100, k = 10, 300 queries; recall " + "vs exact filtered ground truth):\n\n"); + std::printf("| selectivity | allowed ids | recall@10 | mean latency | QPS (1 thread) |\n"); + std::printf("|---|---|---|---|---|\n"); + const std::size_t strides[] = {1000u, 100u, 10u}; // 0.1%, 1%, 10% + const std::size_t nq = 300; + edgevector::SearchContext fctx = graph.make_context(); + for (const std::size_t stride : strides) { + std::vector af((kN + 63u) / 64u, 0u); + std::vector allowed; + for (std::size_t id = 0; id < kN; id += stride) { + af[id >> 6u] |= (1ull << (id & 63u)); + allowed.push_back(static_cast(id)); + } + + std::size_t fhits = 0; + std::vector> brute; + brute.reserve(allowed.size()); + double total_s = 0.0; + for (std::size_t qi = 0; qi < nq; ++qi) { + const std::uint8_t* q = qbase + qi * qstride; + t0 = Clock::now(); + const std::uint32_t found = graph.search( + fctx, q, kK, 100u, results.data(), af.data()); + total_s += seconds_since(t0); + + brute.clear(); + for (const std::uint32_t id : allowed) { + brute.emplace_back( + edgevector::hamming_distance(q, data.record(id), kDim), + id); + } + std::partial_sort(brute.begin(), + brute.begin() + + static_cast(kK), + brute.end()); + for (std::uint32_t i = 0; i < found; ++i) { + got_ids[i] = results[i].id; + } + std::uint32_t truth10[kK]; + for (std::size_t t = 0; t < kK; ++t) { + truth10[t] = brute[t].second; + } + fhits += overlap10(got_ids, found, truth10); + } + std::printf("| %.1f%% | %zu | %.3f | %.0f us | %.0f |\n", + 100.0 / static_cast(stride), allowed.size(), + static_cast(fhits) / + static_cast(nq * kK), + 1.0e6 * total_s / static_cast(nq), + static_cast(nq) / total_s); + std::fflush(stdout); + } + std::printf("\n"); + std::fflush(stdout); + } } // --------------------------------------------------------------------------- diff --git a/tests/test_hnsw_graph.cpp b/tests/test_hnsw_graph.cpp index 0cd6829..640b0bc 100644 --- a/tests/test_hnsw_graph.cpp +++ b/tests/test_hnsw_graph.cpp @@ -488,6 +488,56 @@ void test_delete_and_filter(std::size_t dim) { } } check(composed, "deletion and allow-bitmap compose correctly"); + + // Sparse filters take the exact-scan path: results must EQUAL the + // brute-force top-k over the allowed set, ids and distances both. + std::vector sparse((n + 63u) / 64u, 0u); + std::vector allowed_ids; + for (std::uint32_t id = 4u; id < n; id += 17u) { // ~18 sparse survivors + sparse[id >> 6u] |= (1ull << (id & 63u)); + allowed_ids.push_back(id); + } + found = graph.search(ctx, q, 10u, 100u, res, sparse.data()); + + std::vector> brute; + for (const std::uint32_t id : allowed_ids) { + if (graph.is_deleted(id)) { + continue; + } + brute.emplace_back( + edgevector::hamming_distance(q, data.record(id), dim), id); + } + std::sort(brute.begin(), brute.end()); + bool exact = (found == 10u && brute.size() >= 10u); + for (std::uint32_t i = 0; exact && i < found; ++i) { + exact = (res[i].id == brute[i].second) && + (res[i].distance == brute[i].first); + } + check(exact, + "sparse filter returns the EXACT brute-force top-10 (scan path)"); + + // The re-ranked modes run over the same exact pool. + edgevector::ScoredResult sres[10]; + std::vector qraw(dim, 0.25f); // any float query paired with q bits + const std::uint32_t sf = graph.search_reranked( + ctx, q, qraw.data(), 10u, 100u, sres, sparse.data()); + bool subset = (sf == 10u); + for (std::uint32_t i = 0; subset && i < sf; ++i) { + bool in_allowed = false; + for (const std::uint32_t id : allowed_ids) { + if (sres[i].id == id) { + in_allowed = true; + break; + } + } + subset = in_allowed; + } + check(subset, "re-ranked sparse-filtered results stay within the filter"); + + // An empty filter returns nothing, quickly. + std::vector empty((n + 63u) / 64u, 0u); + check(graph.search(ctx, q, 10u, 100u, res, empty.data()) == 0u, + "empty allow-bitmap returns 0 results"); } // ---------------------------------------------------------------------------