diff --git a/include/edgevector/hnsw_graph.hpp b/include/edgevector/hnsw_graph.hpp index e334125..bd85006 100644 --- a/include/edgevector/hnsw_graph.hpp +++ b/include/edgevector/hnsw_graph.hpp @@ -40,9 +40,17 @@ // memory. Requires the float query alongside the // quantized one. // Both accept an optional caller-owned allow-bitmap (bit id set = id may be -// returned) and always exclude soft-deleted nodes; filtered-out nodes still -// route traversal. NOTE: a highly selective filter degrades toward a scan of -// the reachable graph, as in every filtered-HNSW implementation. +// returned) and always exclude soft-deleted nodes. +// +// FILTERED SEARCH IS SELECTIVITY-ADAPTIVE. A sparse filter makes graph +// traversal the wrong algorithm: too few eligible nodes ever enter the beam, +// pruning never engages, and the walk decays toward scanning the reachable +// graph - expensively and approximately. So when the allowed population is +// small relative to the beam (popcount(allow) <= 16 * ef; the popcount is a +// ~microsecond pass over the bitmap), the search switches to a direct scan +// of the allowed ids, which is both cheaper AND exact - recall 1.0 over the +// allowed set by construction. Larger filters keep the graph traversal, with +// filtered-out nodes still routing. Both paths are zero-allocation. // // DELETION AND SLOT RECLAMATION // ----------------------------- @@ -631,7 +639,7 @@ class HNSWGraph { } clamp_kef(k, ef); - std::uint32_t found = beam_layer0(ctx, q, ef, allow); + std::uint32_t found = collect_pool(ctx, q, ef, allow); while (found > k) { // keep only the k best ctx.res_heap_.pop(); --found; @@ -671,7 +679,7 @@ class HNSWGraph { } clamp_kef(k, ef); - const std::uint32_t found = beam_layer0(ctx, q_bits, ef, allow); + const std::uint32_t found = collect_pool(ctx, q_bits, ef, allow); if (found == 0u) { return 0u; } @@ -738,7 +746,7 @@ class HNSWGraph { } clamp_kef(k, ef); - const std::uint32_t found = beam_layer0(ctx, q_bits, ef, allow); + const std::uint32_t found = collect_pool(ctx, q_bits, ef, allow); if (found == 0u) { return 0u; } @@ -1305,6 +1313,74 @@ class HNSWGraph { return search_layer(ctx, q, ep, ef, 0u, allow, true); } + // Population of an allow-bitmap over the valid id range (bits beyond + // capacity are masked off, whatever the caller left there). + std::uint64_t popcount_allow(const std::uint64_t* allow) const noexcept { + const std::size_t full = static_cast(capacity_) / 64u; + std::uint64_t total = 0u; + for (std::size_t w = 0u; w < full; ++w) { + total += static_cast(__builtin_popcountll(allow[w])); + } + const std::uint32_t rem = capacity_ & 63u; + if (rem != 0u) { + const std::uint64_t mask = (1ull << rem) - 1ull; + total += static_cast( + __builtin_popcountll(allow[full] & mask)); + } + return total; + } + + // Exact scan over the allowed ids: the optimal strategy for sparse + // filters. Fills ctx.res_heap_ with the ef best eligible candidates - + // EXACT over the allowed set, since every allowed id is scored. Ids are + // visited ascending and ties skip the later id, which reproduces the + // (distance, id) selection order of the brute-force baseline exactly. + // Zero allocation. + std::uint32_t filtered_scan(SearchContext& ctx, const std::uint8_t* q, + std::uint32_t ef, + const std::uint64_t* allow) const noexcept { + ctx.res_heap_.clear(); + const std::size_t words = + (static_cast(capacity_) + 63u) / 64u; + const std::uint32_t rem = capacity_ & 63u; + for (std::size_t w = 0u; w < words; ++w) { + std::uint64_t bits = allow[w]; + if (rem != 0u && w == words - 1u) { + bits &= (1ull << rem) - 1ull; + } + while (bits != 0ull) { + const std::uint64_t low = bits & (0ull - bits); + bits ^= low; + const std::uint32_t id = static_cast( + w * 64u + static_cast(__builtin_ctzll(low))); + if (levels_[id] == kNotInserted || is_deleted(id)) { + continue; + } + const std::uint32_t d = distance_to(q, id); + if (ctx.res_heap_.size() < ef) { + ctx.res_heap_.push(detail::Candidate{d, id}); + } else if (d < ctx.res_heap_.top().dist) { + ctx.res_heap_.push(detail::Candidate{d, id}); + ctx.res_heap_.pop(); + } + } + } + return ctx.res_heap_.size(); + } + + // The candidate-pool collector behind all three search modes: picks the + // scan for sparse filters (exact and cheaper), the beam otherwise. + std::uint32_t collect_pool(SearchContext& ctx, const std::uint8_t* q, + std::uint32_t ef, + const std::uint64_t* allow) const noexcept { + if (allow != nullptr && + popcount_allow(allow) <= + static_cast(ef) * 16u) { + return filtered_scan(ctx, q, ef, allow); + } + return beam_layer0(ctx, q, ef, allow); + } + // Beam search on one layer (Algorithm 2), with result-eligibility // filtering: ineligible nodes (tombstoned, or cleared in `allow`) still // route within the beam bound but never enter the result heap. Results diff --git a/tests/benchmark_100k.cpp b/tests/benchmark_100k.cpp index f0eb5cc..0fa1971 100644 --- a/tests/benchmark_100k.cpp +++ b/tests/benchmark_100k.cpp @@ -371,6 +371,67 @@ void run_scenario(const char* name, bool clustered, bool full_report) { } std::printf("\n"); std::fflush(stdout); + + // Filtered search across selectivities (allow every stride-th id). + // Recall is measured against the exact brute-force top-10 over the + // allowed set; sparse filters take the adaptive exact-scan path. + if (full_report) { + std::printf("Filtered search (ef = 100, k = 10, 300 queries; recall " + "vs exact filtered ground truth):\n\n"); + std::printf("| selectivity | allowed ids | recall@10 | mean latency | QPS (1 thread) |\n"); + std::printf("|---|---|---|---|---|\n"); + const std::size_t strides[] = {1000u, 100u, 10u}; // 0.1%, 1%, 10% + const std::size_t nq = 300; + edgevector::SearchContext fctx = graph.make_context(); + for (const std::size_t stride : strides) { + std::vector af((kN + 63u) / 64u, 0u); + std::vector allowed; + for (std::size_t id = 0; id < kN; id += stride) { + af[id >> 6u] |= (1ull << (id & 63u)); + allowed.push_back(static_cast(id)); + } + + std::size_t fhits = 0; + std::vector> brute; + brute.reserve(allowed.size()); + double total_s = 0.0; + for (std::size_t qi = 0; qi < nq; ++qi) { + const std::uint8_t* q = qbase + qi * qstride; + t0 = Clock::now(); + const std::uint32_t found = graph.search( + fctx, q, kK, 100u, results.data(), af.data()); + total_s += seconds_since(t0); + + brute.clear(); + for (const std::uint32_t id : allowed) { + brute.emplace_back( + edgevector::hamming_distance(q, data.record(id), kDim), + id); + } + std::partial_sort(brute.begin(), + brute.begin() + + static_cast(kK), + brute.end()); + for (std::uint32_t i = 0; i < found; ++i) { + got_ids[i] = results[i].id; + } + std::uint32_t truth10[kK]; + for (std::size_t t = 0; t < kK; ++t) { + truth10[t] = brute[t].second; + } + fhits += overlap10(got_ids, found, truth10); + } + std::printf("| %.1f%% | %zu | %.3f | %.0f us | %.0f |\n", + 100.0 / static_cast(stride), allowed.size(), + static_cast(fhits) / + static_cast(nq * kK), + 1.0e6 * total_s / static_cast(nq), + static_cast(nq) / total_s); + std::fflush(stdout); + } + std::printf("\n"); + std::fflush(stdout); + } } // --------------------------------------------------------------------------- diff --git a/tests/test_hnsw_graph.cpp b/tests/test_hnsw_graph.cpp index 0cd6829..640b0bc 100644 --- a/tests/test_hnsw_graph.cpp +++ b/tests/test_hnsw_graph.cpp @@ -488,6 +488,56 @@ void test_delete_and_filter(std::size_t dim) { } } check(composed, "deletion and allow-bitmap compose correctly"); + + // Sparse filters take the exact-scan path: results must EQUAL the + // brute-force top-k over the allowed set, ids and distances both. + std::vector sparse((n + 63u) / 64u, 0u); + std::vector allowed_ids; + for (std::uint32_t id = 4u; id < n; id += 17u) { // ~18 sparse survivors + sparse[id >> 6u] |= (1ull << (id & 63u)); + allowed_ids.push_back(id); + } + found = graph.search(ctx, q, 10u, 100u, res, sparse.data()); + + std::vector> brute; + for (const std::uint32_t id : allowed_ids) { + if (graph.is_deleted(id)) { + continue; + } + brute.emplace_back( + edgevector::hamming_distance(q, data.record(id), dim), id); + } + std::sort(brute.begin(), brute.end()); + bool exact = (found == 10u && brute.size() >= 10u); + for (std::uint32_t i = 0; exact && i < found; ++i) { + exact = (res[i].id == brute[i].second) && + (res[i].distance == brute[i].first); + } + check(exact, + "sparse filter returns the EXACT brute-force top-10 (scan path)"); + + // The re-ranked modes run over the same exact pool. + edgevector::ScoredResult sres[10]; + std::vector qraw(dim, 0.25f); // any float query paired with q bits + const std::uint32_t sf = graph.search_reranked( + ctx, q, qraw.data(), 10u, 100u, sres, sparse.data()); + bool subset = (sf == 10u); + for (std::uint32_t i = 0; subset && i < sf; ++i) { + bool in_allowed = false; + for (const std::uint32_t id : allowed_ids) { + if (sres[i].id == id) { + in_allowed = true; + break; + } + } + subset = in_allowed; + } + check(subset, "re-ranked sparse-filtered results stay within the filter"); + + // An empty filter returns nothing, quickly. + std::vector empty((n + 63u) / 64u, 0u); + check(graph.search(ctx, q, 10u, 100u, res, empty.data()) == 0u, + "empty allow-bitmap returns 0 results"); } // ---------------------------------------------------------------------------